woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

added support for zstd packages

Browse Source
This commit is contained in:
Walter Oggioni
2020-05-04 16:23:21 +01:00
parent a1ef2f6b05
commit ab082860c9
34 changed files with 786 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/net/woggioni/jpacrepo/servlet/FileServlet.java
View File

@@ -1,25 +1,26 @@
package net.woggioni.jpacrepo.servlet;
import net.woggioni.jpacrepo.context.ApplicationContext;
import net.woggioni.jpacrepo.config.AppConfig;
import javax.inject.Inject;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.nio.file.Path;
@WebServlet("/archive/*")
public class FileServlet extends AbstractFileServlet
{
private String root;
private Path root;
@Inject
public FileServlet(ApplicationContext ctx){
public FileServlet(AppConfig ctx){
root = ctx.repoFolder();
}
@Override
protected File getFile(HttpServletRequest request) throws IllegalArgumentException {
return new File(root, request.getPathInfo().substring(1));
return root.resolve(request.getPathInfo().substring(1)).toFile();
}
}

21
src/main/java/net/woggioni/jpacrepo/version/PkgIdComparator.java Normal file
View File

@@ -0,0 +1,21 @@
package net.woggioni.jpacrepo.version;
import net.woggioni.jpacrepo.pacbase.PkgId;
import java.util.Comparator;
public class PkgIdComparator implements Comparator<PkgId> {
private final Comparator<PkgId> comparator;
public PkgIdComparator() {
VersionComparator vc = new VersionComparator();
comparator = Comparator.comparing(PkgId::name)
.thenComparing(PkgId::version, vc)
.thenComparing(PkgId::arch);
}
@Override
public int compare(PkgId id1, PkgId id2) {
return comparator.compare(id1, id2);
}
}

23
src/main/java/net/woggioni/jpacrepo/version/VersionComparator.java Normal file
View File

@@ -0,0 +1,23 @@
package net.woggioni.jpacrepo.version;
import com.sun.jna.Native;
import java.util.Comparator;
class AlpmLibrary {
public static native int alpm_pkg_vercmp(String v2, String v1);
static {
Native.register("alpm");
}
}
public class VersionComparator implements Comparator<String> {
@Override
public int compare(String version1, String version2) {
return AlpmLibrary.alpm_pkg_vercmp(version1, version2);
}
}

5
src/main/resources-template/persistence.xml
View File

@@ -6,6 +6,11 @@
<persistence-unit name="jpacrepo_pu" transaction-type="JTA">
<jta-data-source>${dataSourceJNDI}</jta-data-source>
<class>net.woggioni.jpacrepo.pacbase.PkgData</class>
<class>net.woggioni.jpacrepo.pacbase.PkgId</class>
<exclude-unlisted-classes>true</exclude-unlisted-classes>
<properties>
<property name="org.jboss.logging.provider" value="log4j2"/>
<property name="javax.persistence.schema-generation.database.action" value="${dataBaseAction}"/>

41
src/main/scala/net/woggioni/jpacrepo/config/AppConfig.scala Normal file
View File

@@ -0,0 +1,41 @@
package net.woggioni.jpacrepo.config
import java.nio.file.{Files, Path, Paths}
import java.util.Properties
import java.util.concurrent.atomic.AtomicBoolean
import net.woggioni.jpacrepo.pacbase.PkgData
import net.woggioni.jpacrepo.persistence.InitialSchemaAction
import net.woggioni.jpacrepo.utils.Utils._
object AppConfig {
def apply(propertyFile: String): AppConfig = {
val getProperty = new Properties().let { it =>
val path = Paths.get(propertyFile)
if (Files.exists(path)) {
Files.newInputStream(path).use { is =>
it.load(is)
}
}
(key : String, default : String) => System.getProperty(s"net.woggioni.jpacrepo.$key") match {
case null => it.getProperty(key, default)
case path: String => path
}
}
new AppConfig(
repoFolder = getProperty("RepoFolder", "net.woggioni.jpacrepo.RepoFolder")
.let { it => Paths.get(it) },
initialSchemaAction = getProperty("InitialSchemaAction", "none").let(InitialSchemaAction(_)))
}
}
case class AppConfig(val repoFolder: Path,
val initialSchemaAction: InitialSchemaAction) {
var invalidateCache = new AtomicBoolean(true)
def getFile(pkg: PkgData) = repoFolder.resolve(pkg.fileName)
def getFile(fileName: String) = repoFolder.resolve(fileName)
}

15
src/main/scala/net/woggioni/jpacrepo/config/ApplicationConfig.scala
View File

@@ -1,15 +0,0 @@
package net.woggioni.jpacrepo.config
import javax.ws.rs.ApplicationPath
import javax.ws.rs.core.Application
import net.woggioni.jpacrepo.service.PacmanWebService
import scala.collection.JavaConverters._
@ApplicationPath("rest")
class ApplicationConfig() extends Application {
val classes : Set[Class[_]] = Set(classOf[PacmanWebService])
override def getClasses = classes.asJava
}

42
src/main/scala/net/woggioni/jpacrepo/context/ApplicatioContext.scala
View File

@@ -1,42 +0,0 @@
package net.woggioni.jpacrepo.context
import java.io.{File, FileInputStream}
import java.util.Properties
import javax.enterprise.context.ApplicationScoped
import net.woggioni.jpacrepo.pacbase.PkgData
import net.woggioni.jpacrepo.service.PacmanServiceView
@ApplicationScoped
class ApplicationContext(val propertyFile: String) {
val systemProperties = {
val result = new Properties()
val input = new FileInputStream(propertyFile)
try {
result.load(input)
} finally {
input.close()
}
result
}
val repoFolder = System.getProperty("net.woggioni.jpacrepo.RepoFolder") match {
case null => systemProperties.getProperty("RepoFolder")
case path: String => path
}
private var pacmanService: PacmanServiceView = null
var invalidateCache = true
def getFile(pkg: PkgData) = new File(new File(repoFolder), pkg.fileName)
def getFile(fileName: String) = new File(new File(repoFolder), fileName)
def getPacmanService: PacmanServiceView = pacmanService
def setPacmanService(pacmanService: PacmanServiceView): Unit = {
this.pacmanService = pacmanService
}
}

5
src/main/scala/net/woggioni/jpacrepo/exception/Exceptions.scala Normal file
View File

@@ -0,0 +1,5 @@
package net.woggioni.jpacrepo.exception
class ParseException(msg : String, cause : Throwable) extends RuntimeException(msg, cause) {
def this(msg : String) = this(msg, null)
}

35
src/main/scala/net/woggioni/jpacrepo/factory/BeanFactory.scala
View File

@@ -1,31 +1,42 @@
package net.woggioni.jpacrepo.factory
import javax.ejb.EJB
import java.util.Properties
import javax.annotation.PostConstruct
import javax.enterprise.inject.Produces
import javax.enterprise.inject.spi.InjectionPoint
import javax.persistence.{EntityManagerFactory, PersistenceUnit}
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.service.PacmanServiceView
import javax.faces.bean.ApplicationScoped
import javax.inject.Inject
import javax.persistence.{EntityManagerFactory, Persistence}
import net.woggioni.jpacrepo.config.AppConfig
import org.slf4j.LoggerFactory
class PersistenceUnitFactory {
@PersistenceUnit(unitName = "jpacrepo_pu")
@Inject
private var appConfig : AppConfig = _
private var emf : EntityManagerFactory = _
@PostConstruct
def init(): Unit = {
val properties = new Properties()
properties.put("javax.persistence.schema-generation.database.action", appConfig.initialSchemaAction.value)
emf = Persistence.createEntityManagerFactory("jpacrepo_pu", properties)
}
@Produces
private def createEntityManagerFactory = emf
}
class BeanFactory {
@EJB
private var service : PacmanServiceView = _
@Produces
def produce: ApplicationContext = {
val ctx = new ApplicationContext("/etc/jpacrepo/server.properties")
ctx.setPacmanService(service)
@ApplicationScoped
def produce: AppConfig = {
val ctx = AppConfig(
System.getProperty("net.woggioni.jpacrepo.configuration.file",
"/etc/jpacrepo/server.properties"))
ctx
}
@@ -33,3 +44,5 @@ class BeanFactory {
private def createLogger(injectionPoint: InjectionPoint) =
LoggerFactory.getLogger(injectionPoint.getMember.getDeclaringClass.getName)
}

48
src/main/scala/net/woggioni/jpacrepo/model/Parser.scala
View File

@@ -1,24 +1,37 @@
package net.woggioni.jpacrepo.model
import java.io.{BufferedInputStream, File, FileInputStream}
import java.io.{BufferedInputStream, InputStream}
import java.nio.file.{Files, Path}
import java.util.Date
import java.util.zip.GZIPInputStream
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import org.apache.commons.compress.archivers.ArchiveEntry
import net.woggioni.jpacrepo.exception.ParseException
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData, PkgId}
import net.woggioni.jpacrepo.utils.Utils._
import net.woggioni.jwo.JWO
import net.woggioni.jzstd.ZstdInputStream
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream
import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
import scala.io.Source
object Parser {
def parseFile(file: File): PkgData = {
def parseFile(file: Path, compressionFormat : CompressionFormat): PkgData = {
val hasher = new Hasher("MD5")
val decompressorStreamConstructor = compressionFormat match {
case CompressionFormat.XZ => arg : InputStream => new XZCompressorInputStream(arg)
case CompressionFormat.Z_STANDARD => is : InputStream => ZstdInputStream.from(is)
case CompressionFormat.GZIP => is : InputStream => new GZIPInputStream(is)
case format => throw new ParseException(s"Unsupported compression format '$format'")
}
val is = new TarArchiveInputStream(
new XZCompressorInputStream(
decompressorStreamConstructor(
new BufferedInputStream(
new FileInputStream(file))))
Files.newInputStream(file))))
try {
var archiveEntry = is.getNextEntry
while (archiveEntry != null) {
@@ -33,23 +46,24 @@ object Parser {
.map(line => {
val equals = line.indexOf("=")
if (equals < 0) {
throw new RuntimeException(s"Error parsing .PKGINFO file in '${file}'")
throw new ParseException(s"Error parsing .PKGINFO file in '${file}'")
}
else {
(line.substring(0, equals).trim, line.substring(equals + 1, line.length).trim)
}
})
.toStream
.to(LazyList)
.groupBy(_._1)
.map(pair => pair._1 -> pair._2.map(_._2).toList)
val data = new PkgData
data.id = new PkgId
for (pair <- metadata) {
val (key, value) = pair
key match {
case "size" =>
data.size = value.head.toLong
case "arch" =>
data.arch = value.head
data.id.arch = value.head
case "replaces" =>
data.replaces = value.toSet.asJava
case "packager" =>
@@ -57,17 +71,13 @@ object Parser {
case "url" =>
data.url = value.head
case "pkgname" =>
data.name = {
val name = new PkgName
name.id = value.head
name
}
data.id.name = value.head
case "builddate" =>
data.buildDate = new Date(value.head.toLong * 1000)
case "license" =>
data.license = value.head
case "pkgver" =>
data.version = value.head
data.id.version = value.head
case "pkgdesc" =>
data.description = value.head
case "provides" =>
@@ -89,15 +99,15 @@ object Parser {
case _ =>
}
}
data.md5sum = hasher.getHashString(new FileInputStream(file))
data.fileName = file.getName
data.md5sum = Files.newInputStream(file).use(hasher.getHashString)
data.fileName = file.getFileName().toString
return data
}
else {
archiveEntry = is.getNextEntry()
}
}
throw new RuntimeException(s".PKGINFO file not found in '${file}'")
throw new ParseException(s".PKGINFO file not found in '${file}'")
} finally {
is.close()
}

32
src/main/scala/net/woggioni/jpacrepo/pacbase/CompressionFormat.scala Normal file
View File

@@ -0,0 +1,32 @@
package net.woggioni.jpacrepo.pacbase
import java.nio.file.Path
import net.woggioni.jpacrepo.exception.ParseException
import net.woggioni.jwo.JWO
sealed class CompressionFormat(val extension : String) {
override def toString: String = extension
}
object CompressionFormat {
private val map = LazyList(XZ, GZIP, Z_STANDARD)
.map(v => v.extension -> v).toMap
def apply(text : String) : Option[CompressionFormat] = map.get(text)
def values() = map.values
def guess(file : Path) = {
val extension = JWO.splitExtension(file).orElseThrow(() =>
new ParseException(s"Unable to parse file extension for '${file.getFileName}'")
)._2
CompressionFormat(extension.substring(1)) match {
case Some(compressionFormat) => compressionFormat
case None => throw new IllegalArgumentException(s"Unknown compression format for file extension '$extension'")
}
}
case object XZ extends CompressionFormat("xz")
case object GZIP extends CompressionFormat("gz")
case object Z_STANDARD extends CompressionFormat("zst")
}

8
src/main/scala/net/woggioni/jpacrepo/pacbase/Ordering.scala Normal file
View File

@@ -0,0 +1,8 @@
package net.woggioni.jpacrepo.pacbase
import net.woggioni.jpacrepo.version.VersionComparator
class VersionOrdering extends Ordering[String] {
private val comparator = new VersionComparator
override def compare(x: String, y: String): Int = comparator.compare(x, y)
}

24
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgData.scala
View File

@@ -4,29 +4,29 @@ import java.util
import java.util.Date
import javax.persistence._
import javax.xml.bind.annotation.XmlRootElement
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlRootElement}
@Entity
@Access(AccessType.FIELD)
@XmlRootElement
@NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id")
@NamedQueries(value = Array[NamedQuery](
new NamedQuery(name = "searchByFileName", query = "SELECT p FROM PkgData p WHERE p.fileName = :fileName"),
new NamedQuery(name = "searchByName", query = "SELECT p FROM PkgData p WHERE p.id.name = :name"),
new NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id"),
new NamedQuery(name = "searchByHash", query = "SELECT p FROM PkgData p WHERE p.md5sum = :md5sum")
))
@Table(indexes = Array(
new Index(columnList = "md5sum", unique = true),
new Index(columnList = "fileName", unique = true))
)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
class PkgData {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
var id: Integer = _
@ManyToOne(cascade = Array(CascadeType.PERSIST), fetch = FetchType.EAGER)
var name: PkgName = _
@EmbeddedId
var id: PkgId = _
var base: String = _
var version: String = _
var description: String = _
var url: String = _
@@ -38,8 +38,6 @@ class PkgData {
var size = 0L
var arch: String = _
var license: String = _
var md5sum: String = _

31
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgId.scala Normal file
View File

@@ -0,0 +1,31 @@
package net.woggioni.jpacrepo.pacbase
import javax.persistence.{Access, AccessType, Embeddable}
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlRootElement}
@Embeddable
@Access(AccessType.FIELD)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
class PkgId extends Serializable {
var name : String = null
var version : String = null
var arch : String = null
override def equals(obj: Any): Boolean = {
obj match {
case null => false
case pkgId : PkgId => name == pkgId.name && version == pkgId.version && arch == pkgId.arch
}
}
override def hashCode(): Int = {
var result : Int = 0
if(name != null) result ^= name.hashCode
if(version != null) result ^= version.hashCode
if(arch != null) result ^= arch.hashCode
result
}
}

12
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgName.scala
View File

@@ -1,12 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import javax.persistence.{Access, AccessType, Entity, Id}
import javax.xml.bind.annotation.XmlRootElement
@Entity
@Access(AccessType.FIELD)
@XmlRootElement
class PkgName {
@Id
var id : String = null
}

19
src/main/scala/net/woggioni/jpacrepo/persistence/InitialSchemaAction.scala Normal file
View File

@@ -0,0 +1,19 @@
package net.woggioni.jpacrepo.persistence
sealed class InitialSchemaAction(val value : String) {
override def toString: String = value
}
object InitialSchemaAction {
private val map = LazyList(NONE, CREATE, DROP, DROP_AND_CREATE)
.map(v => v.value -> v).toMap
def apply(text : String) : InitialSchemaAction = map(text)
def values() = map.values
case object NONE extends InitialSchemaAction("none")
case object CREATE extends InitialSchemaAction("create")
case object DROP extends InitialSchemaAction("drop")
case object DROP_AND_CREATE extends InitialSchemaAction("drop-and-create")
}

101
src/main/scala/net/woggioni/jpacrepo/service/PacmanServiceEJB.scala
View File

@@ -1,20 +1,20 @@
package net.woggioni.jpacrepo.service
import java.io.File
import java.nio.file.Files
import java.nio.file.{Files, Path}
import java.util
import java.util.{Calendar, Date}
import javax.annotation.PostConstruct
import javax.ejb._
import javax.inject.Inject
import javax.persistence._
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.config.AppConfig
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData}
import net.woggioni.jpacrepo.persistence.QueryEngine
import org.slf4j.Logger
import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
@Remote
trait PacmanServiceRemote {
@@ -29,7 +29,7 @@ trait PacmanServiceView extends PacmanServiceRemote {
def countResults(name: String, version: String, arch: String): Long
def searchPackage(name: String, version: String, arch: String, page: Int, pageSize: Int, fileName : String): util.List[PkgData]
def searchPackage(name: String, version: String, arch: String, page: Int, pageSize: Int, fileName: String): util.List[PkgData]
}
@Startup
@@ -45,7 +45,7 @@ class PacmanServiceEJB extends PacmanServiceView {
private var emf: EntityManagerFactory = _
@Inject
private var ctx: ApplicationContext = _
private var ctx: AppConfig = _
@Inject
private var logger: Logger = _
@@ -68,7 +68,7 @@ class PacmanServiceEJB extends PacmanServiceView {
@Asynchronous
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Schedule(hour = "10", minute = "34", persistent = false)
@Schedule(hour = "4", minute = "00", persistent = false)
override def syncDB() = {
val em = emf.createEntityManager
logger.info("Starting repository cleanup")
@@ -77,56 +77,55 @@ class PacmanServiceEJB extends PacmanServiceView {
val listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String]).getResultList
logger.info("Got list of filenames from db")
val knownPkg = listaDB
.asScala
.filter(fileName => {
val file = ctx.getFile(fileName)
val result = file.exists()
if (!result) {
logger.info(s"Removing package ${file.getName} which was not found in filesystem")
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
.setParameter("fileName", file.getName)
.getResultList.asScala.foreach((pkgData: PkgData) => deletePkgData(em, pkgData))
}
result
}).toSet
logger.info("Searching for new packages or packages that were modified after being added to the database")
new File(ctx.repoFolder)
.listFiles(_.getName.endsWith(".pkg.tar.xz"))
.foreach(file => {
if (!knownPkg.contains(file.getName) || {
val query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", classOf[Date])
query.setParameter("filename", file.getName)
val result = query.getSingleResult
file.lastModified > result.getTime}) {
try {
parseFile(em, file)
} catch {
case e: Exception =>
logger.error(s"Error parsing '${file.getAbsolutePath}'", e)
}
}
.asScala
.filter(fileName => {
val file = ctx.getFile(fileName)
val result = Files.exists(file)
if (!result) {
logger.info(s"Removing package ${file.getFileName} which was not found in filesystem")
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
.setParameter("fileName", file.getFileName)
.getResultList.asScala.foreach((pkgData: PkgData) => deletePkgData(em, pkgData))
}
)
result
}).toSet
logger.info("Searching for new packages or packages that were modified after being added to the database")
Files.list(ctx.repoFolder).iterator().asScala.filter(file => {
val name = file.getFileName.toString
name.endsWith(".pkg.tar.xz") || name.endsWith(".pkg.tar.zst")
}).foreach(file => {
if (!knownPkg.contains(file.getFileName.toString) || {
val query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", classOf[Date])
query.setParameter("filename", file.getFileName.toString)
val result = query.getSingleResult
Files.getLastModifiedTime(file).toMillis > result.getTime
}) {
try {
parseFile(em, file)
} catch {
case e: Exception =>
logger.error(s"Error parsing '${file.toAbsolutePath}'", e)
if (em.getTransaction.getRollbackOnly) throw e
else Files.delete(file)
}
}
})
logger.info("Removing obsolete packages")
deleteOld(em)
logger.info("Repository cleanup completed successfully")
ctx.invalidateCache = true
ctx.invalidateCache.set(true)
}
private def parseFile(em: EntityManager, file: File) = {
private def parseFile(em: EntityManager, file: Path) = {
val hquery = em.createQuery(hashQueryCount, classOf[java.lang.Long])
val data = Parser.parseFile(file)
val data = Parser.parseFile(file, CompressionFormat.guess(file))
hquery.setParameter("md5sum", data.md5sum)
if(hquery.getSingleResult == 0) {
if (hquery.getSingleResult == 0) {
val fquery = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
fquery.setParameter("fileName", file.getName)
fquery.getResultList.forEach((pkgData: PkgData) => deletePkgData(em, pkgData))
data.name = em.find(classOf[PkgName], data.name.id) match {
case null => data.name
case name : PkgName => name
}
fquery.setParameter("fileName", file.getFileName.toString)
fquery.getResultList.forEach(deletePkgData(em, _))
em.persist(data)
logger.info(s"Persisting package ${file.getName}")
logger.info(s"Persisting package ${file.getFileName}")
}
}
@@ -145,11 +144,11 @@ class PacmanServiceEJB extends PacmanServiceView {
throw new RuntimeException(String.format("Package with name %s not found", filename))
}
val pkg = fquery.getResultList.get(0)
Files.delete(ctx.getFile(pkg).toPath)
Files.delete(ctx.getFile(pkg))
em.remove(pkg)
}
final private val deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" + "(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)"
final private val deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.id.name in \n" + "(SELECT p2.id.name FROM PkgData p2 GROUP BY p2.id.name HAVING count(p2.id.name) > :minVersions\n)"
private def deleteOld(em: EntityManager): Unit = {
val query = em.createQuery(deleteQuery, classOf[String])
@@ -168,7 +167,7 @@ class PacmanServiceEJB extends PacmanServiceView {
}
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
override def searchPackage(name: String, version: String, arch: String, pageNumber: Int, pageSize: Int, fileName : String) = {
override def searchPackage(name: String, version: String, arch: String, pageNumber: Int, pageSize: Int, fileName: String) = {
val em = emf.createEntityManager
QueryEngine.searchPackage(em, name, version, arch, pageNumber, pageSize, null)
}

297
src/main/scala/net/woggioni/jpacrepo/service/PacmanWebService.scala
View File

@@ -2,7 +2,7 @@ package net.woggioni.jpacrepo.service
import java.io._
import java.net.URI
import java.nio.file.Files
import java.nio.file.{Files, Paths}
import java.nio.file.StandardCopyOption.ATOMIC_MOVE
import java.util
@@ -11,57 +11,78 @@ import javax.inject.Inject
import javax.persistence._
import javax.ws.rs._
import javax.ws.rs.core._
import javax.xml.bind.annotation.{XmlElement, XmlRootElement, XmlSeeAlso}
import net.woggioni.jpacrepo.context.ApplicationContext
import javax.xml.bind.annotation.{XmlElement, XmlRootElement}
import net.woggioni.jpacrepo.config.AppConfig
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData, PkgId, VersionOrdering}
import net.woggioni.jpacrepo.utils.Utils._
import net.woggioni.jpacrepo.version.{PkgIdComparator, VersionComparator}
import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
import org.slf4j.Logger
import scala.beans.BeanProperty
import scala.collection.JavaConverters._
import scala.collection.SortedMap
import scala.collection.immutable.TreeMap
import scala.jdk.CollectionConverters._
object PacmanWebService {
@ApplicationPath("api")
class ApplicationConfig() extends Application {
private val nameQuery: String = "SELECT pname FROM PkgName pname WHERE id = :name"
val classes: Set[Class[_]] = Set(classOf[PacmanWebService])
private val fileNameQuery: String = "SELECT pdata FROM PkgData pdata WHERE fileName = :fileName"
private val hashQuery: String = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"
override def getClasses = classes.asJava
}
@XmlRootElement
@XmlSeeAlso(Array(classOf[PkgData]))
class PkgList() extends util.ArrayList[PkgData] {
object PacmanWebService {
val pkgIdOrdering: Ordering[PkgId] = Ordering.comparatorToOrdering(new PkgIdComparator)
val versionOrdering: Ordering[String] = Ordering.comparatorToOrdering(new VersionComparator)
}
def this(c: PkgData*) {
@XmlRootElement
class PkgDataList extends util.ArrayList[PkgData] {
def this(l: util.List[PkgData]) {
this()
c.foreach(el => add(el))
l.forEach(el => add(el))
}
@XmlElement(name = "PkgData")
def getPackages: util.List[PkgData] = this
def this(elements : PkgData*) {
this()
elements.foreach(el => add(el))
}
def setPackages(pkgs: util.List[PkgData]): Unit = {
@XmlElement(name = "pkgData")
def getItems: util.List[PkgData] = this
def setItems(pkgs: util.List[PkgData]): Unit = {
this.clear()
this.addAll(pkgs)
}
}
@XmlRootElement
class StringList() extends util.ArrayList[String] {
class StringList extends util.ArrayList[String] {
def this(c: String*) {
def this(l: util.List[String]) {
this()
c.foreach(el => add(el))
l.forEach(el => add(el))
}
def this(elements : String*) {
this()
elements.foreach(el => add(el))
}
def this(elements : Iterable[String]) {
this()
elements.foreach(el => add(el))
}
@XmlElement(name = "string")
def getPackages: util.List[String] = this
def getItems: util.List[String] = this
def setPackages(pkgs: util.List[String]): Unit = {
def setItems(pkgs: util.List[String]): Unit = {
this.clear()
this.addAll(pkgs)
}
@@ -81,55 +102,67 @@ class PkgTuple {
@Singleton
@Path("/pkg")
@ConcurrencyManagement(ConcurrencyManagementType.BEAN)
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@Consumes(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@TransactionManagement(TransactionManagementType.CONTAINER)
class PacmanWebService {
private var cachedMap: SortedMap[(String, String, String), PkgTuple] = _
private var cachedMap: SortedMap[PkgId, PkgTuple] = _
@Inject
private var emf: EntityManagerFactory = _
@Context
private var uriInfo: UriInfo = _
@Inject
private var log: Logger = _
@Inject
private var service : PacmanServiceView = _
private var service: PacmanServiceView = _
@Inject
private var ctx: ApplicationContext = _
private var ctx: AppConfig = _
private def getCachedMap: SortedMap[(String, String, String), PkgTuple] = {
if (ctx.invalidateCache) {
val em = emf.createEntityManager()
val query = em.createQuery(
"SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.name.id, pkg.version, pkg.arch",
classOf[Array[AnyRef]])
val stream = query.getResultList
.asScala
.toStream
.map(pkg => {
val name: String = pkg(0).asInstanceOf[String]
val version: String = pkg(1).asInstanceOf[String]
val arch: String = pkg(2).asInstanceOf[String]
val filename: String = pkg(3).asInstanceOf[String]
val size: Long = pkg(4).asInstanceOf[Long]
val md5sum: String = pkg(5).asInstanceOf[String]
val tuple: PkgTuple = new PkgTuple
tuple.filename = filename
tuple.size = size
tuple.md5sum = md5sum
(name, version, arch) -> tuple
})
cachedMap = TreeMap(stream: _*)
ctx.invalidateCache = false
private def getCachedMap: SortedMap[PkgId, PkgTuple] = {
var result: SortedMap[PkgId, PkgTuple] = null
if (!ctx.invalidateCache.get()) {
result = cachedMap
}
cachedMap
if (result == null) {
synchronized {
result = cachedMap
if (result == null) {
val em = emf.createEntityManager()
val query = em.createQuery(
"SELECT pkg.id.name, pkg.id.version, pkg.id.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.id.name, pkg.id.version, pkg.id.arch",
classOf[Array[AnyRef]])
val stream = query.getResultList
.asScala
.to(LazyList)
.map(pkg => {
val name: String = pkg(0).asInstanceOf[String]
val version: String = pkg(1).asInstanceOf[String]
val arch: String = pkg(2).asInstanceOf[String]
val filename: String = pkg(3).asInstanceOf[String]
val size: Long = pkg(4).asInstanceOf[Long]
val md5sum: String = pkg(5).asInstanceOf[String]
val tuple: PkgTuple = new PkgTuple
tuple.filename = filename
tuple.size = size
tuple.md5sum = md5sum
val id = new PkgId()
id.name = name
id.version = version
id.arch = arch
id -> tuple
})
cachedMap = TreeMap.from(stream)(PacmanWebService.pkgIdOrdering)
ctx.invalidateCache.set(false)
result = cachedMap
}
}
}
result
}
@GET
@@ -137,7 +170,7 @@ class PacmanWebService {
def searchByName(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST)
val query: String = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name)
val query: String = String.format("SELECT pkgId.name FROM PkgId pkgId WHERE LOWER(pkgId.name) LIKE '%%%s%%' ORDER BY pkgId.name", name)
Response.ok(em.createQuery(query, classOf[String]).getResultList).build
}
@@ -149,26 +182,26 @@ class PacmanWebService {
@Path("list/{name}")
def getPackage(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[_] = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", classOf[String])
val query = em.createQuery("SELECT pkg.id.version FROM PkgData pkg WHERE pkg.id.name = :name ORDER BY pkg.id.version", classOf[String])
query.setParameter("name", name)
Response.ok(query.getResultList).build
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("list/{name}/{version}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[_] = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", classOf[String])
val query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.id.name = :name AND pkg.id.version = :version ORDER BY pkg.id.arch", classOf[String])
query.setParameter("name", name)
query.setParameter("version", version)
Response.ok(query.getResultList).build
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("list/{name}/{version}/{arch}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String, @PathParam("arch") arch: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[PkgData] = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.name.id = :name AND " + "pkg.version = :version AND " + "pkg.arch = :arch " + "ORDER BY pkg.arch", classOf[PkgData])
val query: TypedQuery[PkgData] = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.id.name = :name AND " + "pkg.id.version = :version AND " + "pkg.id.arch = :arch " + "ORDER BY pkg.arch", classOf[PkgData])
query.setParameter("name", name)
query.setParameter("version", version)
query.setParameter("arch", arch)
@@ -183,15 +216,17 @@ class PacmanWebService {
cc.setMaxAge(86400)
cc.setMustRevalidate(true)
cc.setNoCache(true)
val result : util.Map[String, util.Map[String, util.Map[String, PkgTuple]]] = getCachedMap.toStream
.groupBy(_._1._1).mapValues(
_.groupBy(_._1._2).mapValues(
_.map(pair => pair._1._3 -> pair._2).toMap.asJava
).asJava
).asJava
val etag: EntityTag = new EntityTag(Integer.toString(result.hashCode))
val cachedMap = getCachedMap
val etag: EntityTag = new EntityTag(Integer.toString(cachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val result: util.Map[String, util.Map[String, util.Map[String, PkgTuple]]] = cachedMap.to(LazyList)
.groupBy(_._1.name).view.mapValues(t =>
SortedMap.from(t.groupBy(_._1.version).view.mapValues(
_.map(pair => pair._1.arch -> pair._2).to(SortedMap).asJava
))(PacmanWebService.versionOrdering.reverse).asJava
).toMap.asJava
builder = Response.ok(result)
builder.tag(etag)
}
@@ -204,8 +239,7 @@ class PacmanWebService {
def getHashes: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.md5sum FROM PkgData p", classOf[String])
val hl = new StringList(query.getResultList.asScala :_*)
Response.ok(hl).build
Response.ok(new StringList(query.getResultList)).build
}
@GET
@@ -213,20 +247,19 @@ class PacmanWebService {
def getFiles: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String])
val hl = new StringList(query.getResultList.asScala :_*)
Response.ok(hl).build
Response.ok(new StringList(query.getResultList)).build
}
private def getPackageByHash(md5sum: String): Response = {
val em = emf.createEntityManager()
val hquery = em.createQuery(PacmanWebService.hashQuery, classOf[PkgData])
val hquery = em.createNamedQuery("searchByHash", classOf[PkgData])
if (md5sum != null) hquery.setParameter("md5sum", md5sum)
manageQueryResult(hquery.getResultList, true)
}
private def getPackageByFileName(file: String): Response = {
val em = emf.createEntityManager()
val fnquery: TypedQuery[PkgData] = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
val fnquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fnquery.setParameter("fileName", file)
manageQueryResult(fnquery.getResultList, true)
}
@@ -241,9 +274,9 @@ class PacmanWebService {
val etag: EntityTag = new EntityTag(Integer.toString(getCachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val res: File = ctx.getFile(fileName)
if (!res.exists) throw new NotFoundException(String.format("File '%s' was not found", fileName))
builder = Response.ok(res.length)
val res = ctx.getFile(fileName)
if (!Files.exists(res)) throw new NotFoundException(String.format("File '%s' was not found", fileName))
builder = Response.ok(Files.size(res))
builder.tag(etag)
}
builder.cacheControl(cc)
@@ -255,26 +288,24 @@ class PacmanWebService {
@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
def downloadPackage(@PathParam("filename") fileName: String): Response = {
val em = emf.createEntityManager()
val fnquery = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
val fnquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fnquery.setParameter("fileName", fileName)
try {
val pkg: PkgData = fnquery.getSingleResult
val stream: StreamingOutput = (output: OutputStream) => {
val input: FileInputStream = new FileInputStream(ctx.getFile(pkg))
try {
Files.newInputStream(ctx.getFile(pkg)).use { is =>
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ({
read = input.read(bytes)
while ( {
read = is.read(bytes)
read >= 0
}) {
output.write(bytes, 0, read)
}
} finally {
input.close()
}
()
}
return Response.ok(stream).header("Content-Length", ctx.getFile(pkg).length).build
return Response.ok(stream).header("Content-Length", Files.size(ctx.getFile(pkg))).build
} catch {
case _: NoResultException =>
throw new NotFoundException
@@ -286,12 +317,12 @@ class PacmanWebService {
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
def doYouWantAny(filenames: util.List[String]): Response = {
val em = emf.createEntityManager()
val result = Set(filenames.asScala: _*)
val result = Set.from(filenames.asScala)
if (result.nonEmpty) {
val query = em.createQuery("SELECT pkg.fileName from PkgData pkg WHERE pkg.fileName in :filenames", classOf[String])
query.setParameter("filenames", filenames)
val toBeRemoved = Set(query.getResultList.asScala: _*)
Response.ok((result -- toBeRemoved).toArray).build
val toBeRemoved = Set.from(query.getResultList.asScala)
Response.ok(new StringList(result -- toBeRemoved)).build
}
else {
Response.ok(result.toArray).build
@@ -302,40 +333,39 @@ class PacmanWebService {
@Path("/upload")
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Consumes(Array("application/x-xz", "application/gzip", "application/x-tar", MediaType.APPLICATION_OCTET_STREAM))
def createPackage(input: InputStream, @MatrixParam("filename") filename: String): Response = {
def createPackage(input: InputStream,
@MatrixParam("filename") filename: String,
@Context uriInfo: UriInfo): Response = {
val em = emf.createEntityManager()
if (filename == null) throw new BadRequestException
val file: File = File.createTempFile(filename, "tmp", new File(ctx.repoFolder))
val fquery: TypedQuery[PkgData] = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
val file = Files.createTempFile(ctx.repoFolder, filename, null)
val fquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fquery.setParameter("fileName", filename)
val savedFiles: util.List[PkgData] = fquery.getResultList
if (savedFiles.size > 0) Response.notModified.build
else {
val fos: FileOutputStream = new FileOutputStream(file)
val fos: OutputStream = Files.newOutputStream(file)
try {
val buffer: Array[Byte] = new Array[Byte](4096)
val buffer: Array[Byte] = new Array[Byte](0x1000)
var read = 0
while ({
while ( {
read = input.read(buffer)
read >= 0}) {
read >= 0
}) {
fos.write(buffer, 0, read)
}
val pkg = Parser.parseFile(file)
val pkg = Parser.parseFile(file, CompressionFormat.guess(Paths.get(filename)))
pkg.fileName = filename
val nquery: TypedQuery[PkgName] = em.createQuery(PacmanWebService.nameQuery, classOf[PkgName])
nquery.setParameter("name", pkg.name.id)
val savedName: util.List[PkgName] = nquery.getResultList
if (savedName.size > 0) pkg.name = savedName.get(0)
em.persist(pkg)
log.info(s"Persisting package ${pkg.fileName}")
val pkgUri: URI = uriInfo.getAbsolutePathBuilder.path(pkg.fileName).build()
Files.move(file.toPath, new File(ctx.repoFolder, filename).toPath, ATOMIC_MOVE)
ctx.invalidateCache = true
Files.move(file, ctx.repoFolder.resolve(filename), ATOMIC_MOVE)
ctx.invalidateCache.set(true)
cachedMap = null
Response.created(pkgUri).build
} catch {
case e: Exception =>
Files.delete(file.toPath)
Files.delete(file)
throw e
} finally {
fos.close()
@@ -346,13 +376,13 @@ class PacmanWebService {
@GET
@Path("/search")
def searchPackage(
@QueryParam("name") name: String,
@QueryParam("version") version: String,
@QueryParam("arch") arch: String,
@QueryParam("page") pageNumber: Int,
@QueryParam("pageSize") pageSize: Int,
@QueryParam("fileName") fileName: String): util.List[PkgData] = {
service.searchPackage(name, version,arch, pageNumber, pageSize, fileName)
@QueryParam("name") name: String,
@QueryParam("version") version: String,
@QueryParam("arch") arch: String,
@QueryParam("page") pageNumber: Int,
@QueryParam("pageSize") pageSize: Int,
@QueryParam("fileName") fileName: String): util.List[PkgData] = {
service.searchPackage(name, version, arch, pageNumber, pageSize, fileName)
}
@OPTIONS
@@ -366,7 +396,7 @@ class PacmanWebService {
@Consumes(Array(MediaType.APPLICATION_FORM_URLENCODED))
def downloadTar(@FormParam("pkgs") formData: String): Response = {
val files: Array[String] = formData.split(" ")
files.find(!ctx.getFile(_).exists) match {
files.find(fileName => !Files.exists(ctx.getFile(fileName))) match {
case Some(fileName) => throw new NotFoundException(s"Package file '$fileName' does not exist")
case None =>
}
@@ -375,37 +405,40 @@ class PacmanWebService {
val taos: TarArchiveOutputStream = new TarArchiveOutputStream(output)
try {
for (fname <- files) {
val file: File = ctx.getFile(fname)
val input: FileInputStream = new FileInputStream(file)
val entry: TarArchiveEntry = new TarArchiveEntry(fname)
entry.setSize(file.length)
taos.putArchiveEntry(entry)
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ({
read = input.read(bytes)
read >= 0}) {
taos.write(bytes, 0, read)
val file = ctx.getFile(fname)
Files.newInputStream(file).use { input =>
val entry: TarArchiveEntry = new TarArchiveEntry(fname)
entry.setSize(Files.size(file))
taos.putArchiveEntry(entry)
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ( {
read = input.read(bytes)
read >= 0
}) {
taos.write(bytes, 0, read)
}
taos.closeArchiveEntry()
}
taos.closeArchiveEntry()
}
} finally {
taos.close()
}
}
}
return Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build
Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build
}
private def manageQueryResult(list: util.List[PkgData]): Response = {
return manageQueryResult(list, false)
}
private def manageQueryResult(list: util.List[PkgData]): Response = manageQueryResult(list, false)
private def manageQueryResult(list: util.List[PkgData], singleResult: Boolean): Response = {
val pkgList: PkgList = new PkgList(list.asScala :_*)
if (pkgList.size == 0) throw new NotFoundException
else if (singleResult) if (pkgList.size == 1) Response.ok(pkgList.get(0)).build
else throw new NonUniqueResultException("The returned list does not contain a single element")
else Response.ok(pkgList).build
if (list.isEmpty) throw new NotFoundException
else if (singleResult) {
if (list.size == 1) Response.ok(list.get(0)).build
else throw new NonUniqueResultException("The returned list does not contain a single element")
} else {
Response.ok(new PkgDataList(list)).build
}
}
}

22
src/main/scala/net/woggioni/jpacrepo/utils/Utils.scala Normal file
View File

@@ -0,0 +1,22 @@
package net.woggioni.jpacrepo.utils
object Utils {
implicit class Use[CLOSEABLE <: AutoCloseable, RESULT](closeable: CLOSEABLE) {
def use(cb: CLOSEABLE => RESULT): RESULT = {
try {
cb(closeable)
} finally {
closeable.close()
}
}
}
implicit class ContextFunctions[IN, OUT](in: IN) {
def let(cb: (IN) => OUT): OUT = cb(in)
def also(cb: (IN) => Unit) : IN = {
cb(in)
in
}
}
}

42
src/test/java/ClientTest.java
View File

@@ -1,13 +1,14 @@
import com.thoughtworks.xstream.XStream;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.model.Hasher;
import net.woggioni.jpacrepo.model.MD5InputStream;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.PkgData;
import net.woggioni.jpacrepo.service.PacmanServiceRemote;
import net.woggioni.jwo.JWO;
import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
import org.jboss.resteasy.spi.ResteasyProviderFactory;
import org.junit.Test;
import javax.naming.*;
import javax.ws.rs.client.*;
@@ -23,30 +24,39 @@ import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.Properties;
public class ClientTest
{
@Test
public void testGET()
{
public class ClientTest {
private static ObjectMapper om;
static {
om = new ObjectMapper();
JaxbAnnotationModule module = new JaxbAnnotationModule();
om.registerModule(module);
}
@SneakyThrows
public void testGET() {
ResteasyProviderFactory instance = ResteasyProviderFactory.getInstance();
RegisterBuiltin.register(instance);
instance.registerProvider(ResteasyJacksonProvider.class);
// instance.registerProvider(ResteasyJacksonProvider.class);
Client client = ClientBuilder.newClient();
UriBuilder builder = UriBuilder.fromUri("http://odroid-u3:8080/").path("jpacrepo/rest/pkg/search");
UriBuilder builder = UriBuilder.fromUri("http://woggioni.net/").path("jpacrepo/rest/pkg/search");
// builder.queryParam("name", "linux");
// builder.queryParam("version", "324");
builder.queryParam("md5sum", "19787793429AF74D4D2D09890247E2EC");
WebTarget target = client.target(builder.build());
Invocation invocation = target.request().accept("application/xml").buildGet();
Response response = invocation.invoke();
if (response.getStatusInfo() == Response.Status.OK)
{
if (response.getStatusInfo() == Response.Status.OK) {
PkgData pkg = response.readEntity(PkgData.class);
System.out.println(new XStream().toXML(pkg));
om.writeValue(System.out, pkg);
} else {
throw JWO.newThrowable(RuntimeException.class,
"Server returned %d",
response.getStatusInfo().getStatusCode());
}
}
@Test
public void testPUT() throws Exception
{
ResteasyProviderFactory instance = ResteasyProviderFactory.getInstance();
@@ -65,7 +75,6 @@ public class ClientTest
assert Response.Status.CREATED.getStatusCode() == response.getStatus();
}
@Test
public void hashTest() throws Exception
{
String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
@@ -93,7 +102,7 @@ public class ClientTest
h.read(out, 0, (int) a);
System.out.println(h.digest());
PkgData p = Parser.parseFile(new File(file));
PkgData p = Parser.parseFile(Paths.get(file));
System.out.println(p.md5sum());
}
@@ -117,7 +126,6 @@ public class ClientTest
}
}
@Test
public void invokeStatelessBean() throws Exception
{
Properties prop = new Properties();

78
src/test/java/ParseTest.java
View File

@@ -1,46 +1,50 @@
import com.thoughtworks.xstream.XStream;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.CompressionFormat;
import net.woggioni.jpacrepo.pacbase.PkgData;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.junit.Test;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.lang.reflect.TypeVariable;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.regex.Pattern;
/**
* Created by walter on 22/03/15.
*/
public class ParseTest
{
// @Test
public void test()
{
Collection<File> ls = FileUtils.listFiles(new File("/var/cache/pacman/pkg"), new RegexFileFilter(".*\\.pkg\\.tar\\.xz"), DirectoryFileFilter.DIRECTORY);
int i = 0;
List<PkgData> lista = new ArrayList<>();
for (File file : ls)
{
PkgData data = Parser.parseFile(file);
lista.add(data);
//System.out.println(new XStream().toXML(data));
// if(i++>10) break;
}
System.out.print(lista);
public class ParseTest {
@SneakyThrows
public void test() {
ObjectMapper om = new ObjectMapper();
JaxbAnnotationModule module = new JaxbAnnotationModule();
om.registerModule(module);
Pattern pattern = Pattern.compile(".*\\.pkg\\.tar\\.(zst)");
Files.list(Paths.get("/var/cache/pacman/pkg"))
.filter(Files::isRegularFile)
.filter(p -> pattern.matcher(p.getFileName().toString()).matches())
.map(path -> Parser.parseFile(path, CompressionFormat.guess(path)))
.limit(10)
.map(new Function<PkgData, String>() {
@Override
@SneakyThrows
public String apply(PkgData pkgData) {
return om.writeValueAsString(pkgData);
}
})
.forEach(System.out::println);
}
@Test
public void parseTest()
{
String[] files = new String[]{"/home/walter/Scaricati/oh-my-zsh-git-3912.d310fac-1-any.pkg.tar.xz"};
for (String file : files)
{
PkgData data = Parser.parseFile(new File(file));
System.out.println(new XStream().toXML(data));
}
public void test2() {
List<String> l = Arrays.asList("");
Type t = ((ParameterizedType)(l.getClass().getGenericSuperclass())).getActualTypeArguments()[0];
TypeVariable t2 = l.getClass().getTypeParameters()[0];
Type[] bounds = t2.getBounds();
System.out.println(bounds[0]);
System.out.println(bounds[1]);
// GenericEntity<List<String>> entity = new GenericEntity<List<String>>(l, List<String>.getType());
}
}

2
src/test/resources/META-INF/persistence.xml
View File

@@ -7,7 +7,7 @@
<persistence-unit name="test" transaction-type="RESOURCE_LOCAL">
<provider>org.hibernate.jpa.HibernatePersistenceProvider</provider>
<class>net.woggioni.jpacrepo.pacbase.PkgData</class>
<class>net.woggioni.jpacrepo.pacbase.PkgName</class>
<class>net.woggioni.jpacrepo.pacbase.PkgId</class>
<properties>
<property name="javax.persistence.jdbc.driver" value="org.h2.Driver"/>

49
src/test/scala/net/woggioni/jpacrepo/client/SyncDbTest.scala Normal file
View File

@@ -0,0 +1,49 @@
package net.woggioni.jpacrepo.client
import java.io.InputStream
import java.util.Properties
import javax.naming.{Context, InitialContext, NameClassPair, NamingEnumeration, NamingException}
import net.woggioni.jpacrepo.service.PacmanServiceRemote
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
class SyncDbTest extends AnyFlatSpec {
private def traverseJndiNode(nodeName: String, context: Context) {
try {
val list = context.list(nodeName)
while (list.hasMore) {
val childName = nodeName + list.next.getName
System.out.println(childName)
traverseJndiNode(childName, context)
}
} catch {
case _ : NamingException =>
}
}
"it" should "be possible to invoke syncDB remotely" in {
val prop = new Properties
val in = getClass.getClassLoader.getResourceAsStream("jboss-ejb-client.properties")
prop.load(in)
prop.put(Context.URL_PKG_PREFIXES, "org.jboss.ejb.client.naming")
prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory")
prop.put(Context.PROVIDER_URL, "http-remoting://localhost:5080")
// prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
// prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
prop.put(Context.SECURITY_PRINCIPAL, "walter")
prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d")
// prop.put(Context.SECURITY_PRINCIPAL, "admin");
// prop.put(Context.SECURITY_CREDENTIALS, "123456");
prop.put("jboss.naming.client.ejb.context", true)
val context = new InitialContext(prop)
val ctx = new InitialContext(prop)
traverseJndiNode("/", context)
// final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
val service = ctx.lookup("/jpacrepo_2.13-2.0/PacmanServiceEJB!net.woggioni.jpacrepo.service.PacmanServiceRemote").asInstanceOf[PacmanServiceRemote]
// List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
// System.out.println(new XStream().toXML(pkgs));
service.syncDB()
}
}

26
src/test/scala/net/woggioni/jpacrepo/config/AppConfigTest.scala Normal file
View File

@@ -0,0 +1,26 @@
package net.woggioni.jpacrepo.config
import net.woggioni.jpacrepo.factory.BeanFactory
import org.jboss.weld.environment.se.Weld
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.must.Matchers
class AppConfigTest extends AnyFlatSpec with Matchers {
private val weld = new Weld
weld.disableDiscovery()
// weld.alternatives(classOf[TestPersistenceProducer])
weld.beanClasses(
// classOf[PacmanServiceEJB],
// classOf[PacmanWebService],
// classOf[AppConfig],
classOf[BeanFactory],
)
val container = weld.initialize()
"test" should "pass" in {
val appConfig = container.select(classOf[AppConfig]).get()
println(appConfig.repoFolder)
}
}

49
src/test/scala/net/woggioni/jpacrepo/pacbase/MarshalTest.scala Normal file
View File

@@ -0,0 +1,49 @@
package net.woggioni.jpacrepo.pacbase
import java.nio.file.{Files, Path, Paths}
import java.util
import java.util.regex.Pattern
import java.util.stream.Collectors
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlElement, XmlRootElement}
import javax.xml.bind.{JAXBContext, Marshaller}
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.service.PkgDataList
import org.scalatest.flatspec.AnyFlatSpec
import scala.jdk.CollectionConverters._
import scala.annotation.meta.field
@XmlRootElement(name = "List")
@XmlAccessorType(XmlAccessType.FIELD)
class JaxbList2[T] private (@(XmlElement @field)(name = "Item") private val list : util.List[T]) {
def this(s : Seq[T]) {
this(s.toList.asJava)
}
def this() {
this(new util.ArrayList[T]())
}
}
class MarshalTest extends AnyFlatSpec {
"asdfs" should "sdfsd" in {
val context = JAXBContext.newInstance(classOf[PkgId], classOf[PkgDataList])
val mar = context.createMarshaller
mar.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true)
val pkgId2 = new PkgId
pkgId2.name = "linux"
pkgId2.version = "4.6.1"
pkgId2.arch = "x86_64"
val pattern = Pattern.compile(".*\\.pkg\\.tar\\.(zst)")
val pkgDatas = Files.list(Paths.get("/var/cache/pacman/pkg"))
.filter(Files.isRegularFile(_))
.filter((p: Path) => pattern.matcher(p.getFileName.toString).matches)
.limit(10)
.map(Parser.parseFile)
.collect(Collectors.toList[PkgData])
val list = new PkgDataList(pkgDatas)
mar.marshal(list, System.out)
}
}

31
src/test/scala/net/woggioni/jpacrepo/sample/ExampleSpec.scala Normal file
View File

@@ -0,0 +1,31 @@
package net.woggioni.jpacrepo.sample
import net.woggioni.jpacrepo.persistence.InitialSchemaAction
import net.woggioni.jpacrepo.version.VersionComparator
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import scala.collection.mutable
class ExampleSpec extends AnyFlatSpec with Matchers {
// "A Stack" should "pop values in last-in-first-out order" in {
// val stack = new mutable.Stack[Int]
// stack.push(1)
// stack.push(2)
// stack.pop() should be (2)
// stack.pop() should be (1)
// }
//
// it should "throw NoSuchElementException if an empty stack is popped" in {
// val emptyStack = new mutable.Stack[Int]
// a [NoSuchElementException] should be thrownBy {
// emptyStack.pop()
// }
// }
"sdfgf" should "dfgfd" in {
val initial = InitialSchemaAction("none")
println(initial)
}
}

3
src/test/scala/net/woggioni/jpacrepo/service/PacmanServiceEJBTest.scala
View File

@@ -1,11 +1,9 @@
package net.woggioni.jpacrepo.service
import javax.enterprise.util.TypeLiteral
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.factory.BeanFactory
import net.woggioni.jpacrepo.persistence.TestPersistenceProducer
import org.jboss.weld.environment.se.Weld
import org.junit.Test
//object WeldContainer {
// private val weld = new Weld
@@ -47,7 +45,6 @@ class PacmanServiceEJBTest {
classOf[BeanFactory],
)
@Test
def test = {
val container = weld.initialize()
try {

4
src/test/scala/net/woggioni/jpacrepo/service/PacmanWebServiceTest.scala
View File

@@ -26,4 +26,8 @@ class PacmanWebServiceTest {
// val res = response.getEntity
// response.getStatus
// }
def boh = {
println(System.getProperty("net.woggioni.jpacrepo.configuration.file"))
}
}

16
src/test/scala/net/woggioni/jpacrepo/version/VersionComparatorSpec.scala Normal file
View File

@@ -0,0 +1,16 @@
package net.woggioni.jpacrepo.version
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import scala.util.Random
class VersionComparatorSpec extends AnyFlatSpec with Matchers {
"Version sorting" should "work as expected" in {
val vc = new VersionComparator
val originalList = List("1.0", "2.0", "3.0", "5.6.7.arch1-1", "5.6.7.arch3-1", "5.6.7.arch3-2", "20200421.78c0348-1")
val shuffledList = new Random(101325).shuffle(originalList)
val sortedList = shuffledList.sorted(Ordering.comparatorToOrdering(vc))
sortedList should be (originalList)
}
}