added support for zstd packages

This commit is contained in:
2020-05-04 16:23:21 +01:00
parent a1ef2f6b05
commit ab082860c9
34 changed files with 786 additions and 406 deletions

View File

@@ -3,51 +3,35 @@ import org.oggio88.sbt.ConfigurationFile._
name := "jpacrepo"
organization := "com.oggio88"
organization := "net.woggioni"
version := "2.0"
resolvers += Resolver.mavenLocal
libraryDependencies += "org.tukaani" % "xz" % "1.6"
libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.25"
libraryDependencies += "org.hibernate" % "hibernate-jpamodelgen" % "5.4.2.Final" % Provided
libraryDependencies += "org.apache.commons" % "commons-compress" % "1.14"
libraryDependencies += "org.projectlombok" % "lombok" % "1.18.8" % Provided
libraryDependencies += "javax" % "javaee-api" % "7.0" % Provided
scalaVersion := "2.13.2"
libraryDependencies += "junit" % "junit" % "4.12" % Test
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test
libraryDependencies += "com.thoughtworks.xstream" % "xstream" % "1.4.10" % Test
libraryDependencies += "commons-io" % "commons-io" % "2.5" % Test
libraryDependencies += "org.jboss" % "jboss-ejb-client" % "4.0.18.Final" % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.11.2" % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-core" % "2.11.2" % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-api" % "2.11.2" % Test
libraryDependencies += "org.tukaani" % "xz" % Versions.xz
libraryDependencies += "org.slf4j" % "slf4j-api" % Versions.slf4j
libraryDependencies += "net.woggioni" % "jzstd" % Versions.jzstd
libraryDependencies += "net.woggioni" % "jwo" % Versions.jwo
libraryDependencies += "org.apache.commons" % "commons-compress" % Versions.`common-compress`
//libraryDependencies += "org.jboss.resteasy" % "resteasy-undertow" % "3.0.4.Final" % Test
//libraryDependencies += "io.undertow" % "undertow-core" % "2.0.20.Final" % Test
//libraryDependencies += "io.undertow" % "undertow-servlet" % "2.0.20.Final" % Test
libraryDependencies += "org.hibernate" % "hibernate-jpamodelgen" % Versions.hibernate % Provided
libraryDependencies += "org.projectlombok" % "lombok" % Versions.lombok % Provided
libraryDependencies += "javax" % "javaee-api" % Versions.javaee % Provided
//libraryDependencies += "org.jboss.weld.servlet" % "weld-servlet-core" % "3.1.1.Final" % Test
libraryDependencies += "org.jboss.weld.se" % "weld-se-core" % "3.1.1.Final" % Test
libraryDependencies += "com.h2database" % "h2" % "1.4.197" % Test
libraryDependencies += "org.hibernate" % "hibernate-core" % "5.3.6.Final" % Test
libraryDependencies += "org.jboss" % "jboss-ejb-client" % Versions.jbossEjbClient % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j % Test
libraryDependencies += "org.jboss.weld.se" % "weld-se-core" % Versions.weld % Test
libraryDependencies += "com.h2database" % "h2" % Versions.h2 % Test
libraryDependencies += "org.hibernate" % "hibernate-core" % Versions.hibernate % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-client" % Versions.restEasy % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-jackson2-provider" % Versions.restEasy % Test
libraryDependencies += "org.scalatest" %% "scalatest" % Versions.scalatest % Test
//libraryDependencies += "io.smallrye" % "smallrye-config" % "1.3.5" % Test
//libraryDependencies += "org.jboss.logging" % "jboss-logging-processor" % "2.2.0.Final" % Test
//libraryDependencies += "org.eclipse.microprofile.config" % "microprofile-config-api" % "1.3" % Test
//libraryDependencies += "org.jboss.spec.javax.ws.rs" % "jboss-jaxrs-api_2.1_spec" % "1.0.2.Final" % Test
//libraryDependencies += "org.reactivestreams" % "reactive-streams" % "1.0.2" % Test
//libraryDependencies += "org.jboss.spec.javax.xml.bind" % "jboss-jaxb-api_2.3_spec" % "1.0.1.Final" % Test
//libraryDependencies += "javax.activation" % "activation" % "1.1.1" % Test
//libraryDependencies += "javax.validation" % "validation-api" % "2.0.1.Final" % Test
//
//libraryDependencies += "org.jboss.resteasy" % "resteasy-cdi" % "4.0.0.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-jaxrs" % "3.0.11.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-client" % "3.0.11.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-jackson-provider" % "3.0.11.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-jaxb-provider" % "3.0.11.Final" % Test
enablePlugins(WarPlugin)
enablePlugins(WildflyPlugin)
@@ -56,21 +40,24 @@ enablePlugins(NimPlugin)
nimCompilerParameters in CompileNim := Seq("-d:release", "-d:serverURL=")
sources in CompileNim := Seq(baseDirectory.value / "nim" / "src" / "jpacrepo.nim")
//fork in Test := true
webappWebInfClasses := true
Test / run / javaOptions += "-Dnet.woggioni.jpacrepo.configuration.file=conf/server.properties"
Test / run / fork := true
webappPostProcess := {
val baseDir = baseDirectory.value / "nim" / "static"
val nimOutput = (compileNim in CompileNim).value
println(nimOutput)
webappDir: File =>
{
webappDir: File => {
IO.copyFile(baseDir / "index.html", webappDir / "index.html")
IO.copyFile(baseDir / "jpacrepo.css", webappDir / "jpacrepo.css")
nimOutput.foreach(file => IO.copyFile(file, webappDir / file.getName))
}
}
lazy val datasourceJNDI = SettingKey[String]("datasource-jndi", "JNDI name of the application datasource")
lazy val databaseAction = SettingKey[String]("database-action",
"value of the property \"javax.persistence.schema-generation.database.action\" in the persistence unit")

1
conf/server.properties Normal file
View File

@@ -0,0 +1 @@
Repofolder=/var/cache/pacman/pkg

View File

@@ -10,7 +10,7 @@ from sequtils import map, apply
var pkgMap : JsonNode
const serverURL {.strdefine.}: string = "http://oggio88.soon.it/jpacrepo/"
const serverURL {.strdefine.}: string = "http://woggioni.net/jpacrepo/"
proc last[T](s : seq[T]) : T = s[s.len - 1]
proc formatByteSize(size : BiggestInt) : string = size.float64.formatEng(precision=1, siPrefix=true, unit = "B")
@@ -70,7 +70,7 @@ proc newDownloadPanel(parent : Element) : DownloadPanel =
let form = cast[Formelement](document.createElement("form"))
form.style.display = "none"
form.setAttribute("method", "post")
form.setAttribute("action", serverURL & "rest/pkg/downloadTar")
form.setAttribute("action", serverURL & "api/pkg/downloadTar")
let tf= document.createElement("input")
tf.setAttribute("name", "pkgs")
let txt = sequtils.foldl(pkglist, a & " " & b)
@@ -120,7 +120,7 @@ proc addPkg(dp : DownloadPanel, pkgfile : string) =
cb:
listElement = elem
req.addEventListener("load", load_cb)
req.open("get", serverURL & "rest/pkg/filesize/" & pkgfile)
req.open("get", serverURL & "api/pkg/filesize/" & pkgfile)
req.setRequestHeader("Accept", "application/json")
req.send()
dp.updateBadge
@@ -141,7 +141,7 @@ proc createDropdown(parent : Element, data :seq[string], onchange : proc(value :
classList = ["btn", "btn-default", "dropdown-toggle"]
attrs = {"data-toggle": "dropdown", "type": "button"}
cb:
elem.textContent = data.last
elem.textContent = data[0]
button = elem
"ul":
classList = ["dropdown-menu"]
@@ -165,11 +165,11 @@ proc newPkgTable(parent: Element, searchString : string) : PkgTable =
var fragments = newSeq[string]()
for fragment in searchString.splitWhitespace():
fragments.add(fragment)
var searchResult = newOrderedTable[string,JsonNode]()
var searchResult = newOrderedTable[string, JsonNode]()
for key, value in pkgMap:
for fragment in fragments:
if fragment in key:
searchResult.add(key,value)
searchResult.add(key, value)
for table in document.querySelectorAll("table.pkgtable"):
table.parentNode.removeChild(table)
htmlTreeAppend(parent):
@@ -206,7 +206,7 @@ proc newPkgTable(parent: Element, searchString : string) : PkgTable =
var archCell : Element
var sizeCell : Element
let size_change_callback = proc(newValue : string) =
sizeCell.textContent = readTableRow(row)["size"].getNum.formatByteSize
sizeCell.textContent = readTableRow(row)["size"].getInt.formatByteSize
"td":
"div":
@@ -245,7 +245,7 @@ proc newPkgTable(parent: Element, searchString : string) : PkgTable =
sizeCell = elem
for v, arches in versions:
for key, value in arches:
elem.textContent = value["size"].getNum.formatByteSize
elem.textContent = value["size"].getInt.formatByteSize
return
cb:
row = elem
@@ -320,6 +320,6 @@ let r = newXMLHTTPRequest()
let load_cb = proc(e : Event) =
pkgMap = parseJson($r.responseText)
r.addEventListener("load", load_cb)
r.open("get", serverURL & "rest/pkg/map")
r.open("get", serverURL & "api/pkg/map")
r.setRequestHeader("Accept", "application/json")
r.send()

View File

@@ -1 +1 @@
sbt.version=1.3.1
sbt.version=1.3.9

16
project/build.scala Normal file
View File

@@ -0,0 +1,16 @@
object Versions {
val restEasy = "4.5.3.Final"
val hibernate = "5.4.15.Final"
val jwo = "1.0"
val jzstd = "0.1"
val slf4j = "1.7.30"
val log4j = "2.13.2"
val javaee = "7.0"
val xz = "1.8"
val h2 = "1.4.200"
val weld = "3.1.4.Final"
val jbossEjbClient = "4.0.32.Final"
val scalatest = "3.2.0-M4"
val lombok = "1.18.8"
val `common-compress` = "1.14"
}

View File

@@ -1,25 +1,26 @@
package net.woggioni.jpacrepo.servlet;
import net.woggioni.jpacrepo.context.ApplicationContext;
import net.woggioni.jpacrepo.config.AppConfig;
import javax.inject.Inject;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.nio.file.Path;
@WebServlet("/archive/*")
public class FileServlet extends AbstractFileServlet
{
private String root;
private Path root;
@Inject
public FileServlet(ApplicationContext ctx){
public FileServlet(AppConfig ctx){
root = ctx.repoFolder();
}
@Override
protected File getFile(HttpServletRequest request) throws IllegalArgumentException {
return new File(root, request.getPathInfo().substring(1));
return root.resolve(request.getPathInfo().substring(1)).toFile();
}
}

View File

@@ -0,0 +1,21 @@
package net.woggioni.jpacrepo.version;
import net.woggioni.jpacrepo.pacbase.PkgId;
import java.util.Comparator;
public class PkgIdComparator implements Comparator<PkgId> {
private final Comparator<PkgId> comparator;
public PkgIdComparator() {
VersionComparator vc = new VersionComparator();
comparator = Comparator.comparing(PkgId::name)
.thenComparing(PkgId::version, vc)
.thenComparing(PkgId::arch);
}
@Override
public int compare(PkgId id1, PkgId id2) {
return comparator.compare(id1, id2);
}
}

View File

@@ -0,0 +1,23 @@
package net.woggioni.jpacrepo.version;
import com.sun.jna.Native;
import java.util.Comparator;
class AlpmLibrary {
public static native int alpm_pkg_vercmp(String v2, String v1);
static {
Native.register("alpm");
}
}
public class VersionComparator implements Comparator<String> {
@Override
public int compare(String version1, String version2) {
return AlpmLibrary.alpm_pkg_vercmp(version1, version2);
}
}

View File

@@ -6,6 +6,11 @@
<persistence-unit name="jpacrepo_pu" transaction-type="JTA">
<jta-data-source>${dataSourceJNDI}</jta-data-source>
<class>net.woggioni.jpacrepo.pacbase.PkgData</class>
<class>net.woggioni.jpacrepo.pacbase.PkgId</class>
<exclude-unlisted-classes>true</exclude-unlisted-classes>
<properties>
<property name="org.jboss.logging.provider" value="log4j2"/>
<property name="javax.persistence.schema-generation.database.action" value="${dataBaseAction}"/>

View File

@@ -0,0 +1,41 @@
package net.woggioni.jpacrepo.config
import java.nio.file.{Files, Path, Paths}
import java.util.Properties
import java.util.concurrent.atomic.AtomicBoolean
import net.woggioni.jpacrepo.pacbase.PkgData
import net.woggioni.jpacrepo.persistence.InitialSchemaAction
import net.woggioni.jpacrepo.utils.Utils._
object AppConfig {
def apply(propertyFile: String): AppConfig = {
val getProperty = new Properties().let { it =>
val path = Paths.get(propertyFile)
if (Files.exists(path)) {
Files.newInputStream(path).use { is =>
it.load(is)
}
}
(key : String, default : String) => System.getProperty(s"net.woggioni.jpacrepo.$key") match {
case null => it.getProperty(key, default)
case path: String => path
}
}
new AppConfig(
repoFolder = getProperty("RepoFolder", "net.woggioni.jpacrepo.RepoFolder")
.let { it => Paths.get(it) },
initialSchemaAction = getProperty("InitialSchemaAction", "none").let(InitialSchemaAction(_)))
}
}
case class AppConfig(val repoFolder: Path,
val initialSchemaAction: InitialSchemaAction) {
var invalidateCache = new AtomicBoolean(true)
def getFile(pkg: PkgData) = repoFolder.resolve(pkg.fileName)
def getFile(fileName: String) = repoFolder.resolve(fileName)
}

View File

@@ -1,15 +0,0 @@
package net.woggioni.jpacrepo.config
import javax.ws.rs.ApplicationPath
import javax.ws.rs.core.Application
import net.woggioni.jpacrepo.service.PacmanWebService
import scala.collection.JavaConverters._
@ApplicationPath("rest")
class ApplicationConfig() extends Application {
val classes : Set[Class[_]] = Set(classOf[PacmanWebService])
override def getClasses = classes.asJava
}

View File

@@ -1,42 +0,0 @@
package net.woggioni.jpacrepo.context
import java.io.{File, FileInputStream}
import java.util.Properties
import javax.enterprise.context.ApplicationScoped
import net.woggioni.jpacrepo.pacbase.PkgData
import net.woggioni.jpacrepo.service.PacmanServiceView
@ApplicationScoped
class ApplicationContext(val propertyFile: String) {
val systemProperties = {
val result = new Properties()
val input = new FileInputStream(propertyFile)
try {
result.load(input)
} finally {
input.close()
}
result
}
val repoFolder = System.getProperty("net.woggioni.jpacrepo.RepoFolder") match {
case null => systemProperties.getProperty("RepoFolder")
case path: String => path
}
private var pacmanService: PacmanServiceView = null
var invalidateCache = true
def getFile(pkg: PkgData) = new File(new File(repoFolder), pkg.fileName)
def getFile(fileName: String) = new File(new File(repoFolder), fileName)
def getPacmanService: PacmanServiceView = pacmanService
def setPacmanService(pacmanService: PacmanServiceView): Unit = {
this.pacmanService = pacmanService
}
}

View File

@@ -0,0 +1,5 @@
package net.woggioni.jpacrepo.exception
class ParseException(msg : String, cause : Throwable) extends RuntimeException(msg, cause) {
def this(msg : String) = this(msg, null)
}

View File

@@ -1,31 +1,42 @@
package net.woggioni.jpacrepo.factory
import javax.ejb.EJB
import java.util.Properties
import javax.annotation.PostConstruct
import javax.enterprise.inject.Produces
import javax.enterprise.inject.spi.InjectionPoint
import javax.persistence.{EntityManagerFactory, PersistenceUnit}
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.service.PacmanServiceView
import javax.faces.bean.ApplicationScoped
import javax.inject.Inject
import javax.persistence.{EntityManagerFactory, Persistence}
import net.woggioni.jpacrepo.config.AppConfig
import org.slf4j.LoggerFactory
class PersistenceUnitFactory {
@PersistenceUnit(unitName = "jpacrepo_pu")
@Inject
private var appConfig : AppConfig = _
private var emf : EntityManagerFactory = _
@PostConstruct
def init(): Unit = {
val properties = new Properties()
properties.put("javax.persistence.schema-generation.database.action", appConfig.initialSchemaAction.value)
emf = Persistence.createEntityManagerFactory("jpacrepo_pu", properties)
}
@Produces
private def createEntityManagerFactory = emf
}
class BeanFactory {
@EJB
private var service : PacmanServiceView = _
@Produces
def produce: ApplicationContext = {
val ctx = new ApplicationContext("/etc/jpacrepo/server.properties")
ctx.setPacmanService(service)
@ApplicationScoped
def produce: AppConfig = {
val ctx = AppConfig(
System.getProperty("net.woggioni.jpacrepo.configuration.file",
"/etc/jpacrepo/server.properties"))
ctx
}
@@ -33,3 +44,5 @@ class BeanFactory {
private def createLogger(injectionPoint: InjectionPoint) =
LoggerFactory.getLogger(injectionPoint.getMember.getDeclaringClass.getName)
}

View File

@@ -1,24 +1,37 @@
package net.woggioni.jpacrepo.model
import java.io.{BufferedInputStream, File, FileInputStream}
import java.io.{BufferedInputStream, InputStream}
import java.nio.file.{Files, Path}
import java.util.Date
import java.util.zip.GZIPInputStream
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import org.apache.commons.compress.archivers.ArchiveEntry
import net.woggioni.jpacrepo.exception.ParseException
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData, PkgId}
import net.woggioni.jpacrepo.utils.Utils._
import net.woggioni.jwo.JWO
import net.woggioni.jzstd.ZstdInputStream
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream
import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
import scala.io.Source
object Parser {
def parseFile(file: File): PkgData = {
def parseFile(file: Path, compressionFormat : CompressionFormat): PkgData = {
val hasher = new Hasher("MD5")
val decompressorStreamConstructor = compressionFormat match {
case CompressionFormat.XZ => arg : InputStream => new XZCompressorInputStream(arg)
case CompressionFormat.Z_STANDARD => is : InputStream => ZstdInputStream.from(is)
case CompressionFormat.GZIP => is : InputStream => new GZIPInputStream(is)
case format => throw new ParseException(s"Unsupported compression format '$format'")
}
val is = new TarArchiveInputStream(
new XZCompressorInputStream(
decompressorStreamConstructor(
new BufferedInputStream(
new FileInputStream(file))))
Files.newInputStream(file))))
try {
var archiveEntry = is.getNextEntry
while (archiveEntry != null) {
@@ -33,23 +46,24 @@ object Parser {
.map(line => {
val equals = line.indexOf("=")
if (equals < 0) {
throw new RuntimeException(s"Error parsing .PKGINFO file in '${file}'")
throw new ParseException(s"Error parsing .PKGINFO file in '${file}'")
}
else {
(line.substring(0, equals).trim, line.substring(equals + 1, line.length).trim)
}
})
.toStream
.to(LazyList)
.groupBy(_._1)
.map(pair => pair._1 -> pair._2.map(_._2).toList)
val data = new PkgData
data.id = new PkgId
for (pair <- metadata) {
val (key, value) = pair
key match {
case "size" =>
data.size = value.head.toLong
case "arch" =>
data.arch = value.head
data.id.arch = value.head
case "replaces" =>
data.replaces = value.toSet.asJava
case "packager" =>
@@ -57,17 +71,13 @@ object Parser {
case "url" =>
data.url = value.head
case "pkgname" =>
data.name = {
val name = new PkgName
name.id = value.head
name
}
data.id.name = value.head
case "builddate" =>
data.buildDate = new Date(value.head.toLong * 1000)
case "license" =>
data.license = value.head
case "pkgver" =>
data.version = value.head
data.id.version = value.head
case "pkgdesc" =>
data.description = value.head
case "provides" =>
@@ -89,15 +99,15 @@ object Parser {
case _ =>
}
}
data.md5sum = hasher.getHashString(new FileInputStream(file))
data.fileName = file.getName
data.md5sum = Files.newInputStream(file).use(hasher.getHashString)
data.fileName = file.getFileName().toString
return data
}
else {
archiveEntry = is.getNextEntry()
}
}
throw new RuntimeException(s".PKGINFO file not found in '${file}'")
throw new ParseException(s".PKGINFO file not found in '${file}'")
} finally {
is.close()
}

View File

@@ -0,0 +1,32 @@
package net.woggioni.jpacrepo.pacbase
import java.nio.file.Path
import net.woggioni.jpacrepo.exception.ParseException
import net.woggioni.jwo.JWO
sealed class CompressionFormat(val extension : String) {
override def toString: String = extension
}
object CompressionFormat {
private val map = LazyList(XZ, GZIP, Z_STANDARD)
.map(v => v.extension -> v).toMap
def apply(text : String) : Option[CompressionFormat] = map.get(text)
def values() = map.values
def guess(file : Path) = {
val extension = JWO.splitExtension(file).orElseThrow(() =>
new ParseException(s"Unable to parse file extension for '${file.getFileName}'")
)._2
CompressionFormat(extension.substring(1)) match {
case Some(compressionFormat) => compressionFormat
case None => throw new IllegalArgumentException(s"Unknown compression format for file extension '$extension'")
}
}
case object XZ extends CompressionFormat("xz")
case object GZIP extends CompressionFormat("gz")
case object Z_STANDARD extends CompressionFormat("zst")
}

View File

@@ -0,0 +1,8 @@
package net.woggioni.jpacrepo.pacbase
import net.woggioni.jpacrepo.version.VersionComparator
class VersionOrdering extends Ordering[String] {
private val comparator = new VersionComparator
override def compare(x: String, y: String): Int = comparator.compare(x, y)
}

View File

@@ -4,29 +4,29 @@ import java.util
import java.util.Date
import javax.persistence._
import javax.xml.bind.annotation.XmlRootElement
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlRootElement}
@Entity
@Access(AccessType.FIELD)
@XmlRootElement
@NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id")
@NamedQueries(value = Array[NamedQuery](
new NamedQuery(name = "searchByFileName", query = "SELECT p FROM PkgData p WHERE p.fileName = :fileName"),
new NamedQuery(name = "searchByName", query = "SELECT p FROM PkgData p WHERE p.id.name = :name"),
new NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id"),
new NamedQuery(name = "searchByHash", query = "SELECT p FROM PkgData p WHERE p.md5sum = :md5sum")
))
@Table(indexes = Array(
new Index(columnList = "md5sum", unique = true),
new Index(columnList = "fileName", unique = true))
)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
class PkgData {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
var id: Integer = _
@ManyToOne(cascade = Array(CascadeType.PERSIST), fetch = FetchType.EAGER)
var name: PkgName = _
@EmbeddedId
var id: PkgId = _
var base: String = _
var version: String = _
var description: String = _
var url: String = _
@@ -38,8 +38,6 @@ class PkgData {
var size = 0L
var arch: String = _
var license: String = _
var md5sum: String = _

View File

@@ -0,0 +1,31 @@
package net.woggioni.jpacrepo.pacbase
import javax.persistence.{Access, AccessType, Embeddable}
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlRootElement}
@Embeddable
@Access(AccessType.FIELD)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
class PkgId extends Serializable {
var name : String = null
var version : String = null
var arch : String = null
override def equals(obj: Any): Boolean = {
obj match {
case null => false
case pkgId : PkgId => name == pkgId.name && version == pkgId.version && arch == pkgId.arch
}
}
override def hashCode(): Int = {
var result : Int = 0
if(name != null) result ^= name.hashCode
if(version != null) result ^= version.hashCode
if(arch != null) result ^= arch.hashCode
result
}
}

View File

@@ -1,12 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import javax.persistence.{Access, AccessType, Entity, Id}
import javax.xml.bind.annotation.XmlRootElement
@Entity
@Access(AccessType.FIELD)
@XmlRootElement
class PkgName {
@Id
var id : String = null
}

View File

@@ -0,0 +1,19 @@
package net.woggioni.jpacrepo.persistence
sealed class InitialSchemaAction(val value : String) {
override def toString: String = value
}
object InitialSchemaAction {
private val map = LazyList(NONE, CREATE, DROP, DROP_AND_CREATE)
.map(v => v.value -> v).toMap
def apply(text : String) : InitialSchemaAction = map(text)
def values() = map.values
case object NONE extends InitialSchemaAction("none")
case object CREATE extends InitialSchemaAction("create")
case object DROP extends InitialSchemaAction("drop")
case object DROP_AND_CREATE extends InitialSchemaAction("drop-and-create")
}

View File

@@ -1,20 +1,20 @@
package net.woggioni.jpacrepo.service
import java.io.File
import java.nio.file.Files
import java.nio.file.{Files, Path}
import java.util
import java.util.{Calendar, Date}
import javax.annotation.PostConstruct
import javax.ejb._
import javax.inject.Inject
import javax.persistence._
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.config.AppConfig
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData}
import net.woggioni.jpacrepo.persistence.QueryEngine
import org.slf4j.Logger
import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
@Remote
trait PacmanServiceRemote {
@@ -29,7 +29,7 @@ trait PacmanServiceView extends PacmanServiceRemote {
def countResults(name: String, version: String, arch: String): Long
def searchPackage(name: String, version: String, arch: String, page: Int, pageSize: Int, fileName : String): util.List[PkgData]
def searchPackage(name: String, version: String, arch: String, page: Int, pageSize: Int, fileName: String): util.List[PkgData]
}
@Startup
@@ -45,7 +45,7 @@ class PacmanServiceEJB extends PacmanServiceView {
private var emf: EntityManagerFactory = _
@Inject
private var ctx: ApplicationContext = _
private var ctx: AppConfig = _
@Inject
private var logger: Logger = _
@@ -68,7 +68,7 @@ class PacmanServiceEJB extends PacmanServiceView {
@Asynchronous
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Schedule(hour = "10", minute = "34", persistent = false)
@Schedule(hour = "4", minute = "00", persistent = false)
override def syncDB() = {
val em = emf.createEntityManager
logger.info("Starting repository cleanup")
@@ -80,53 +80,52 @@ class PacmanServiceEJB extends PacmanServiceView {
.asScala
.filter(fileName => {
val file = ctx.getFile(fileName)
val result = file.exists()
val result = Files.exists(file)
if (!result) {
logger.info(s"Removing package ${file.getName} which was not found in filesystem")
logger.info(s"Removing package ${file.getFileName} which was not found in filesystem")
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
.setParameter("fileName", file.getName)
.setParameter("fileName", file.getFileName)
.getResultList.asScala.foreach((pkgData: PkgData) => deletePkgData(em, pkgData))
}
result
}).toSet
logger.info("Searching for new packages or packages that were modified after being added to the database")
new File(ctx.repoFolder)
.listFiles(_.getName.endsWith(".pkg.tar.xz"))
.foreach(file => {
if (!knownPkg.contains(file.getName) || {
Files.list(ctx.repoFolder).iterator().asScala.filter(file => {
val name = file.getFileName.toString
name.endsWith(".pkg.tar.xz") || name.endsWith(".pkg.tar.zst")
}).foreach(file => {
if (!knownPkg.contains(file.getFileName.toString) || {
val query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", classOf[Date])
query.setParameter("filename", file.getName)
query.setParameter("filename", file.getFileName.toString)
val result = query.getSingleResult
file.lastModified > result.getTime}) {
Files.getLastModifiedTime(file).toMillis > result.getTime
}) {
try {
parseFile(em, file)
} catch {
case e: Exception =>
logger.error(s"Error parsing '${file.getAbsolutePath}'", e)
logger.error(s"Error parsing '${file.toAbsolutePath}'", e)
if (em.getTransaction.getRollbackOnly) throw e
else Files.delete(file)
}
}
}
)
})
logger.info("Removing obsolete packages")
deleteOld(em)
logger.info("Repository cleanup completed successfully")
ctx.invalidateCache = true
ctx.invalidateCache.set(true)
}
private def parseFile(em: EntityManager, file: File) = {
private def parseFile(em: EntityManager, file: Path) = {
val hquery = em.createQuery(hashQueryCount, classOf[java.lang.Long])
val data = Parser.parseFile(file)
val data = Parser.parseFile(file, CompressionFormat.guess(file))
hquery.setParameter("md5sum", data.md5sum)
if(hquery.getSingleResult == 0) {
if (hquery.getSingleResult == 0) {
val fquery = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
fquery.setParameter("fileName", file.getName)
fquery.getResultList.forEach((pkgData: PkgData) => deletePkgData(em, pkgData))
data.name = em.find(classOf[PkgName], data.name.id) match {
case null => data.name
case name : PkgName => name
}
fquery.setParameter("fileName", file.getFileName.toString)
fquery.getResultList.forEach(deletePkgData(em, _))
em.persist(data)
logger.info(s"Persisting package ${file.getName}")
logger.info(s"Persisting package ${file.getFileName}")
}
}
@@ -145,11 +144,11 @@ class PacmanServiceEJB extends PacmanServiceView {
throw new RuntimeException(String.format("Package with name %s not found", filename))
}
val pkg = fquery.getResultList.get(0)
Files.delete(ctx.getFile(pkg).toPath)
Files.delete(ctx.getFile(pkg))
em.remove(pkg)
}
final private val deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" + "(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)"
final private val deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.id.name in \n" + "(SELECT p2.id.name FROM PkgData p2 GROUP BY p2.id.name HAVING count(p2.id.name) > :minVersions\n)"
private def deleteOld(em: EntityManager): Unit = {
val query = em.createQuery(deleteQuery, classOf[String])
@@ -168,7 +167,7 @@ class PacmanServiceEJB extends PacmanServiceView {
}
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
override def searchPackage(name: String, version: String, arch: String, pageNumber: Int, pageSize: Int, fileName : String) = {
override def searchPackage(name: String, version: String, arch: String, pageNumber: Int, pageSize: Int, fileName: String) = {
val em = emf.createEntityManager
QueryEngine.searchPackage(em, name, version, arch, pageNumber, pageSize, null)
}

View File

@@ -2,7 +2,7 @@ package net.woggioni.jpacrepo.service
import java.io._
import java.net.URI
import java.nio.file.Files
import java.nio.file.{Files, Paths}
import java.nio.file.StandardCopyOption.ATOMIC_MOVE
import java.util
@@ -11,57 +11,78 @@ import javax.inject.Inject
import javax.persistence._
import javax.ws.rs._
import javax.ws.rs.core._
import javax.xml.bind.annotation.{XmlElement, XmlRootElement, XmlSeeAlso}
import net.woggioni.jpacrepo.context.ApplicationContext
import javax.xml.bind.annotation.{XmlElement, XmlRootElement}
import net.woggioni.jpacrepo.config.AppConfig
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData, PkgId, VersionOrdering}
import net.woggioni.jpacrepo.utils.Utils._
import net.woggioni.jpacrepo.version.{PkgIdComparator, VersionComparator}
import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
import org.slf4j.Logger
import scala.beans.BeanProperty
import scala.collection.JavaConverters._
import scala.collection.SortedMap
import scala.collection.immutable.TreeMap
import scala.jdk.CollectionConverters._
object PacmanWebService {
@ApplicationPath("api")
class ApplicationConfig() extends Application {
private val nameQuery: String = "SELECT pname FROM PkgName pname WHERE id = :name"
val classes: Set[Class[_]] = Set(classOf[PacmanWebService])
private val fileNameQuery: String = "SELECT pdata FROM PkgData pdata WHERE fileName = :fileName"
private val hashQuery: String = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"
override def getClasses = classes.asJava
}
@XmlRootElement
@XmlSeeAlso(Array(classOf[PkgData]))
class PkgList() extends util.ArrayList[PkgData] {
object PacmanWebService {
val pkgIdOrdering: Ordering[PkgId] = Ordering.comparatorToOrdering(new PkgIdComparator)
val versionOrdering: Ordering[String] = Ordering.comparatorToOrdering(new VersionComparator)
}
def this(c: PkgData*) {
@XmlRootElement
class PkgDataList extends util.ArrayList[PkgData] {
def this(l: util.List[PkgData]) {
this()
c.foreach(el => add(el))
l.forEach(el => add(el))
}
@XmlElement(name = "PkgData")
def getPackages: util.List[PkgData] = this
def this(elements : PkgData*) {
this()
elements.foreach(el => add(el))
}
def setPackages(pkgs: util.List[PkgData]): Unit = {
@XmlElement(name = "pkgData")
def getItems: util.List[PkgData] = this
def setItems(pkgs: util.List[PkgData]): Unit = {
this.clear()
this.addAll(pkgs)
}
}
@XmlRootElement
class StringList() extends util.ArrayList[String] {
class StringList extends util.ArrayList[String] {
def this(c: String*) {
def this(l: util.List[String]) {
this()
c.foreach(el => add(el))
l.forEach(el => add(el))
}
def this(elements : String*) {
this()
elements.foreach(el => add(el))
}
def this(elements : Iterable[String]) {
this()
elements.foreach(el => add(el))
}
@XmlElement(name = "string")
def getPackages: util.List[String] = this
def getItems: util.List[String] = this
def setPackages(pkgs: util.List[String]): Unit = {
def setItems(pkgs: util.List[String]): Unit = {
this.clear()
this.addAll(pkgs)
}
@@ -81,38 +102,43 @@ class PkgTuple {
@Singleton
@Path("/pkg")
@ConcurrencyManagement(ConcurrencyManagementType.BEAN)
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@Consumes(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@TransactionManagement(TransactionManagementType.CONTAINER)
class PacmanWebService {
private var cachedMap: SortedMap[(String, String, String), PkgTuple] = _
private var cachedMap: SortedMap[PkgId, PkgTuple] = _
@Inject
private var emf: EntityManagerFactory = _
@Context
private var uriInfo: UriInfo = _
@Inject
private var log: Logger = _
@Inject
private var service : PacmanServiceView = _
private var service: PacmanServiceView = _
@Inject
private var ctx: ApplicationContext = _
private var ctx: AppConfig = _
private def getCachedMap: SortedMap[(String, String, String), PkgTuple] = {
if (ctx.invalidateCache) {
private def getCachedMap: SortedMap[PkgId, PkgTuple] = {
var result: SortedMap[PkgId, PkgTuple] = null
if (!ctx.invalidateCache.get()) {
result = cachedMap
}
if (result == null) {
synchronized {
result = cachedMap
if (result == null) {
val em = emf.createEntityManager()
val query = em.createQuery(
"SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.name.id, pkg.version, pkg.arch",
"SELECT pkg.id.name, pkg.id.version, pkg.id.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.id.name, pkg.id.version, pkg.id.arch",
classOf[Array[AnyRef]])
val stream = query.getResultList
.asScala
.toStream
.to(LazyList)
.map(pkg => {
val name: String = pkg(0).asInstanceOf[String]
val version: String = pkg(1).asInstanceOf[String]
@@ -124,12 +150,19 @@ class PacmanWebService {
tuple.filename = filename
tuple.size = size
tuple.md5sum = md5sum
(name, version, arch) -> tuple
val id = new PkgId()
id.name = name
id.version = version
id.arch = arch
id -> tuple
})
cachedMap = TreeMap(stream: _*)
ctx.invalidateCache = false
cachedMap = TreeMap.from(stream)(PacmanWebService.pkgIdOrdering)
ctx.invalidateCache.set(false)
result = cachedMap
}
cachedMap
}
}
result
}
@GET
@@ -137,7 +170,7 @@ class PacmanWebService {
def searchByName(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST)
val query: String = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name)
val query: String = String.format("SELECT pkgId.name FROM PkgId pkgId WHERE LOWER(pkgId.name) LIKE '%%%s%%' ORDER BY pkgId.name", name)
Response.ok(em.createQuery(query, classOf[String]).getResultList).build
}
@@ -149,26 +182,26 @@ class PacmanWebService {
@Path("list/{name}")
def getPackage(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[_] = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", classOf[String])
val query = em.createQuery("SELECT pkg.id.version FROM PkgData pkg WHERE pkg.id.name = :name ORDER BY pkg.id.version", classOf[String])
query.setParameter("name", name)
Response.ok(query.getResultList).build
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("list/{name}/{version}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[_] = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", classOf[String])
val query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.id.name = :name AND pkg.id.version = :version ORDER BY pkg.id.arch", classOf[String])
query.setParameter("name", name)
query.setParameter("version", version)
Response.ok(query.getResultList).build
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("list/{name}/{version}/{arch}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String, @PathParam("arch") arch: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[PkgData] = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.name.id = :name AND " + "pkg.version = :version AND " + "pkg.arch = :arch " + "ORDER BY pkg.arch", classOf[PkgData])
val query: TypedQuery[PkgData] = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.id.name = :name AND " + "pkg.id.version = :version AND " + "pkg.id.arch = :arch " + "ORDER BY pkg.arch", classOf[PkgData])
query.setParameter("name", name)
query.setParameter("version", version)
query.setParameter("arch", arch)
@@ -183,15 +216,17 @@ class PacmanWebService {
cc.setMaxAge(86400)
cc.setMustRevalidate(true)
cc.setNoCache(true)
val result : util.Map[String, util.Map[String, util.Map[String, PkgTuple]]] = getCachedMap.toStream
.groupBy(_._1._1).mapValues(
_.groupBy(_._1._2).mapValues(
_.map(pair => pair._1._3 -> pair._2).toMap.asJava
).asJava
).asJava
val etag: EntityTag = new EntityTag(Integer.toString(result.hashCode))
val cachedMap = getCachedMap
val etag: EntityTag = new EntityTag(Integer.toString(cachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val result: util.Map[String, util.Map[String, util.Map[String, PkgTuple]]] = cachedMap.to(LazyList)
.groupBy(_._1.name).view.mapValues(t =>
SortedMap.from(t.groupBy(_._1.version).view.mapValues(
_.map(pair => pair._1.arch -> pair._2).to(SortedMap).asJava
))(PacmanWebService.versionOrdering.reverse).asJava
).toMap.asJava
builder = Response.ok(result)
builder.tag(etag)
}
@@ -204,8 +239,7 @@ class PacmanWebService {
def getHashes: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.md5sum FROM PkgData p", classOf[String])
val hl = new StringList(query.getResultList.asScala :_*)
Response.ok(hl).build
Response.ok(new StringList(query.getResultList)).build
}
@GET
@@ -213,20 +247,19 @@ class PacmanWebService {
def getFiles: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String])
val hl = new StringList(query.getResultList.asScala :_*)
Response.ok(hl).build
Response.ok(new StringList(query.getResultList)).build
}
private def getPackageByHash(md5sum: String): Response = {
val em = emf.createEntityManager()
val hquery = em.createQuery(PacmanWebService.hashQuery, classOf[PkgData])
val hquery = em.createNamedQuery("searchByHash", classOf[PkgData])
if (md5sum != null) hquery.setParameter("md5sum", md5sum)
manageQueryResult(hquery.getResultList, true)
}
private def getPackageByFileName(file: String): Response = {
val em = emf.createEntityManager()
val fnquery: TypedQuery[PkgData] = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
val fnquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fnquery.setParameter("fileName", file)
manageQueryResult(fnquery.getResultList, true)
}
@@ -241,9 +274,9 @@ class PacmanWebService {
val etag: EntityTag = new EntityTag(Integer.toString(getCachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val res: File = ctx.getFile(fileName)
if (!res.exists) throw new NotFoundException(String.format("File '%s' was not found", fileName))
builder = Response.ok(res.length)
val res = ctx.getFile(fileName)
if (!Files.exists(res)) throw new NotFoundException(String.format("File '%s' was not found", fileName))
builder = Response.ok(Files.size(res))
builder.tag(etag)
}
builder.cacheControl(cc)
@@ -255,26 +288,24 @@ class PacmanWebService {
@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
def downloadPackage(@PathParam("filename") fileName: String): Response = {
val em = emf.createEntityManager()
val fnquery = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
val fnquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fnquery.setParameter("fileName", fileName)
try {
val pkg: PkgData = fnquery.getSingleResult
val stream: StreamingOutput = (output: OutputStream) => {
val input: FileInputStream = new FileInputStream(ctx.getFile(pkg))
try {
Files.newInputStream(ctx.getFile(pkg)).use { is =>
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ({
read = input.read(bytes)
while ( {
read = is.read(bytes)
read >= 0
}) {
output.write(bytes, 0, read)
}
} finally {
input.close()
}
()
}
return Response.ok(stream).header("Content-Length", ctx.getFile(pkg).length).build
return Response.ok(stream).header("Content-Length", Files.size(ctx.getFile(pkg))).build
} catch {
case _: NoResultException =>
throw new NotFoundException
@@ -286,12 +317,12 @@ class PacmanWebService {
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
def doYouWantAny(filenames: util.List[String]): Response = {
val em = emf.createEntityManager()
val result = Set(filenames.asScala: _*)
val result = Set.from(filenames.asScala)
if (result.nonEmpty) {
val query = em.createQuery("SELECT pkg.fileName from PkgData pkg WHERE pkg.fileName in :filenames", classOf[String])
query.setParameter("filenames", filenames)
val toBeRemoved = Set(query.getResultList.asScala: _*)
Response.ok((result -- toBeRemoved).toArray).build
val toBeRemoved = Set.from(query.getResultList.asScala)
Response.ok(new StringList(result -- toBeRemoved)).build
}
else {
Response.ok(result.toArray).build
@@ -302,40 +333,39 @@ class PacmanWebService {
@Path("/upload")
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Consumes(Array("application/x-xz", "application/gzip", "application/x-tar", MediaType.APPLICATION_OCTET_STREAM))
def createPackage(input: InputStream, @MatrixParam("filename") filename: String): Response = {
def createPackage(input: InputStream,
@MatrixParam("filename") filename: String,
@Context uriInfo: UriInfo): Response = {
val em = emf.createEntityManager()
if (filename == null) throw new BadRequestException
val file: File = File.createTempFile(filename, "tmp", new File(ctx.repoFolder))
val fquery: TypedQuery[PkgData] = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
val file = Files.createTempFile(ctx.repoFolder, filename, null)
val fquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fquery.setParameter("fileName", filename)
val savedFiles: util.List[PkgData] = fquery.getResultList
if (savedFiles.size > 0) Response.notModified.build
else {
val fos: FileOutputStream = new FileOutputStream(file)
val fos: OutputStream = Files.newOutputStream(file)
try {
val buffer: Array[Byte] = new Array[Byte](4096)
val buffer: Array[Byte] = new Array[Byte](0x1000)
var read = 0
while ({
while ( {
read = input.read(buffer)
read >= 0}) {
read >= 0
}) {
fos.write(buffer, 0, read)
}
val pkg = Parser.parseFile(file)
val pkg = Parser.parseFile(file, CompressionFormat.guess(Paths.get(filename)))
pkg.fileName = filename
val nquery: TypedQuery[PkgName] = em.createQuery(PacmanWebService.nameQuery, classOf[PkgName])
nquery.setParameter("name", pkg.name.id)
val savedName: util.List[PkgName] = nquery.getResultList
if (savedName.size > 0) pkg.name = savedName.get(0)
em.persist(pkg)
log.info(s"Persisting package ${pkg.fileName}")
val pkgUri: URI = uriInfo.getAbsolutePathBuilder.path(pkg.fileName).build()
Files.move(file.toPath, new File(ctx.repoFolder, filename).toPath, ATOMIC_MOVE)
ctx.invalidateCache = true
Files.move(file, ctx.repoFolder.resolve(filename), ATOMIC_MOVE)
ctx.invalidateCache.set(true)
cachedMap = null
Response.created(pkgUri).build
} catch {
case e: Exception =>
Files.delete(file.toPath)
Files.delete(file)
throw e
} finally {
fos.close()
@@ -352,7 +382,7 @@ class PacmanWebService {
@QueryParam("page") pageNumber: Int,
@QueryParam("pageSize") pageSize: Int,
@QueryParam("fileName") fileName: String): util.List[PkgData] = {
service.searchPackage(name, version,arch, pageNumber, pageSize, fileName)
service.searchPackage(name, version, arch, pageNumber, pageSize, fileName)
}
@OPTIONS
@@ -366,7 +396,7 @@ class PacmanWebService {
@Consumes(Array(MediaType.APPLICATION_FORM_URLENCODED))
def downloadTar(@FormParam("pkgs") formData: String): Response = {
val files: Array[String] = formData.split(" ")
files.find(!ctx.getFile(_).exists) match {
files.find(fileName => !Files.exists(ctx.getFile(fileName))) match {
case Some(fileName) => throw new NotFoundException(s"Package file '$fileName' does not exist")
case None =>
}
@@ -375,37 +405,40 @@ class PacmanWebService {
val taos: TarArchiveOutputStream = new TarArchiveOutputStream(output)
try {
for (fname <- files) {
val file: File = ctx.getFile(fname)
val input: FileInputStream = new FileInputStream(file)
val file = ctx.getFile(fname)
Files.newInputStream(file).use { input =>
val entry: TarArchiveEntry = new TarArchiveEntry(fname)
entry.setSize(file.length)
entry.setSize(Files.size(file))
taos.putArchiveEntry(entry)
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ({
while ( {
read = input.read(bytes)
read >= 0}) {
read >= 0
}) {
taos.write(bytes, 0, read)
}
taos.closeArchiveEntry()
}
}
} finally {
taos.close()
}
}
}
return Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build
Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build
}
private def manageQueryResult(list: util.List[PkgData]): Response = {
return manageQueryResult(list, false)
}
private def manageQueryResult(list: util.List[PkgData]): Response = manageQueryResult(list, false)
private def manageQueryResult(list: util.List[PkgData], singleResult: Boolean): Response = {
val pkgList: PkgList = new PkgList(list.asScala :_*)
if (pkgList.size == 0) throw new NotFoundException
else if (singleResult) if (pkgList.size == 1) Response.ok(pkgList.get(0)).build
if (list.isEmpty) throw new NotFoundException
else if (singleResult) {
if (list.size == 1) Response.ok(list.get(0)).build
else throw new NonUniqueResultException("The returned list does not contain a single element")
else Response.ok(pkgList).build
} else {
Response.ok(new PkgDataList(list)).build
}
}
}

View File

@@ -0,0 +1,22 @@
package net.woggioni.jpacrepo.utils
object Utils {
implicit class Use[CLOSEABLE <: AutoCloseable, RESULT](closeable: CLOSEABLE) {
def use(cb: CLOSEABLE => RESULT): RESULT = {
try {
cb(closeable)
} finally {
closeable.close()
}
}
}
implicit class ContextFunctions[IN, OUT](in: IN) {
def let(cb: (IN) => OUT): OUT = cb(in)
def also(cb: (IN) => Unit) : IN = {
cb(in)
in
}
}
}

View File

@@ -1,13 +1,14 @@
import com.thoughtworks.xstream.XStream;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.model.Hasher;
import net.woggioni.jpacrepo.model.MD5InputStream;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.PkgData;
import net.woggioni.jpacrepo.service.PacmanServiceRemote;
import net.woggioni.jwo.JWO;
import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
import org.jboss.resteasy.spi.ResteasyProviderFactory;
import org.junit.Test;
import javax.naming.*;
import javax.ws.rs.client.*;
@@ -23,30 +24,39 @@ import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.Properties;
public class ClientTest
{
@Test
public void testGET()
{
public class ClientTest {
private static ObjectMapper om;
static {
om = new ObjectMapper();
JaxbAnnotationModule module = new JaxbAnnotationModule();
om.registerModule(module);
}
@SneakyThrows
public void testGET() {
ResteasyProviderFactory instance = ResteasyProviderFactory.getInstance();
RegisterBuiltin.register(instance);
instance.registerProvider(ResteasyJacksonProvider.class);
// instance.registerProvider(ResteasyJacksonProvider.class);
Client client = ClientBuilder.newClient();
UriBuilder builder = UriBuilder.fromUri("http://odroid-u3:8080/").path("jpacrepo/rest/pkg/search");
UriBuilder builder = UriBuilder.fromUri("http://woggioni.net/").path("jpacrepo/rest/pkg/search");
// builder.queryParam("name", "linux");
// builder.queryParam("version", "324");
builder.queryParam("md5sum", "19787793429AF74D4D2D09890247E2EC");
WebTarget target = client.target(builder.build());
Invocation invocation = target.request().accept("application/xml").buildGet();
Response response = invocation.invoke();
if (response.getStatusInfo() == Response.Status.OK)
{
if (response.getStatusInfo() == Response.Status.OK) {
PkgData pkg = response.readEntity(PkgData.class);
System.out.println(new XStream().toXML(pkg));
om.writeValue(System.out, pkg);
} else {
throw JWO.newThrowable(RuntimeException.class,
"Server returned %d",
response.getStatusInfo().getStatusCode());
}
}
@Test
public void testPUT() throws Exception
{
ResteasyProviderFactory instance = ResteasyProviderFactory.getInstance();
@@ -65,7 +75,6 @@ public class ClientTest
assert Response.Status.CREATED.getStatusCode() == response.getStatus();
}
@Test
public void hashTest() throws Exception
{
String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
@@ -93,7 +102,7 @@ public class ClientTest
h.read(out, 0, (int) a);
System.out.println(h.digest());
PkgData p = Parser.parseFile(new File(file));
PkgData p = Parser.parseFile(Paths.get(file));
System.out.println(p.md5sum());
}
@@ -117,7 +126,6 @@ public class ClientTest
}
}
@Test
public void invokeStatelessBean() throws Exception
{
Properties prop = new Properties();

View File

@@ -1,46 +1,50 @@
import com.thoughtworks.xstream.XStream;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.CompressionFormat;
import net.woggioni.jpacrepo.pacbase.PkgData;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.junit.Test;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.lang.reflect.TypeVariable;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.regex.Pattern;
/**
* Created by walter on 22/03/15.
*/
public class ParseTest
{
// @Test
public void test()
{
Collection<File> ls = FileUtils.listFiles(new File("/var/cache/pacman/pkg"), new RegexFileFilter(".*\\.pkg\\.tar\\.xz"), DirectoryFileFilter.DIRECTORY);
int i = 0;
List<PkgData> lista = new ArrayList<>();
for (File file : ls)
{
PkgData data = Parser.parseFile(file);
lista.add(data);
//System.out.println(new XStream().toXML(data));
// if(i++>10) break;
public class ParseTest {
@SneakyThrows
public void test() {
ObjectMapper om = new ObjectMapper();
JaxbAnnotationModule module = new JaxbAnnotationModule();
om.registerModule(module);
Pattern pattern = Pattern.compile(".*\\.pkg\\.tar\\.(zst)");
Files.list(Paths.get("/var/cache/pacman/pkg"))
.filter(Files::isRegularFile)
.filter(p -> pattern.matcher(p.getFileName().toString()).matches())
.map(path -> Parser.parseFile(path, CompressionFormat.guess(path)))
.limit(10)
.map(new Function<PkgData, String>() {
@Override
@SneakyThrows
public String apply(PkgData pkgData) {
return om.writeValueAsString(pkgData);
}
System.out.print(lista);
})
.forEach(System.out::println);
}
@Test
public void parseTest()
{
String[] files = new String[]{"/home/walter/Scaricati/oh-my-zsh-git-3912.d310fac-1-any.pkg.tar.xz"};
for (String file : files)
{
PkgData data = Parser.parseFile(new File(file));
System.out.println(new XStream().toXML(data));
}
public void test2() {
List<String> l = Arrays.asList("");
Type t = ((ParameterizedType)(l.getClass().getGenericSuperclass())).getActualTypeArguments()[0];
TypeVariable t2 = l.getClass().getTypeParameters()[0];
Type[] bounds = t2.getBounds();
System.out.println(bounds[0]);
System.out.println(bounds[1]);
// GenericEntity<List<String>> entity = new GenericEntity<List<String>>(l, List<String>.getType());
}
}

View File

@@ -7,7 +7,7 @@
<persistence-unit name="test" transaction-type="RESOURCE_LOCAL">
<provider>org.hibernate.jpa.HibernatePersistenceProvider</provider>
<class>net.woggioni.jpacrepo.pacbase.PkgData</class>
<class>net.woggioni.jpacrepo.pacbase.PkgName</class>
<class>net.woggioni.jpacrepo.pacbase.PkgId</class>
<properties>
<property name="javax.persistence.jdbc.driver" value="org.h2.Driver"/>

View File

@@ -0,0 +1,49 @@
package net.woggioni.jpacrepo.client
import java.io.InputStream
import java.util.Properties
import javax.naming.{Context, InitialContext, NameClassPair, NamingEnumeration, NamingException}
import net.woggioni.jpacrepo.service.PacmanServiceRemote
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
class SyncDbTest extends AnyFlatSpec {
private def traverseJndiNode(nodeName: String, context: Context) {
try {
val list = context.list(nodeName)
while (list.hasMore) {
val childName = nodeName + list.next.getName
System.out.println(childName)
traverseJndiNode(childName, context)
}
} catch {
case _ : NamingException =>
}
}
"it" should "be possible to invoke syncDB remotely" in {
val prop = new Properties
val in = getClass.getClassLoader.getResourceAsStream("jboss-ejb-client.properties")
prop.load(in)
prop.put(Context.URL_PKG_PREFIXES, "org.jboss.ejb.client.naming")
prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory")
prop.put(Context.PROVIDER_URL, "http-remoting://localhost:5080")
// prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
// prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
prop.put(Context.SECURITY_PRINCIPAL, "walter")
prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d")
// prop.put(Context.SECURITY_PRINCIPAL, "admin");
// prop.put(Context.SECURITY_CREDENTIALS, "123456");
prop.put("jboss.naming.client.ejb.context", true)
val context = new InitialContext(prop)
val ctx = new InitialContext(prop)
traverseJndiNode("/", context)
// final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
val service = ctx.lookup("/jpacrepo_2.13-2.0/PacmanServiceEJB!net.woggioni.jpacrepo.service.PacmanServiceRemote").asInstanceOf[PacmanServiceRemote]
// List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
// System.out.println(new XStream().toXML(pkgs));
service.syncDB()
}
}

View File

@@ -0,0 +1,26 @@
package net.woggioni.jpacrepo.config
import net.woggioni.jpacrepo.factory.BeanFactory
import org.jboss.weld.environment.se.Weld
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.must.Matchers
class AppConfigTest extends AnyFlatSpec with Matchers {
private val weld = new Weld
weld.disableDiscovery()
// weld.alternatives(classOf[TestPersistenceProducer])
weld.beanClasses(
// classOf[PacmanServiceEJB],
// classOf[PacmanWebService],
// classOf[AppConfig],
classOf[BeanFactory],
)
val container = weld.initialize()
"test" should "pass" in {
val appConfig = container.select(classOf[AppConfig]).get()
println(appConfig.repoFolder)
}
}

View File

@@ -0,0 +1,49 @@
package net.woggioni.jpacrepo.pacbase
import java.nio.file.{Files, Path, Paths}
import java.util
import java.util.regex.Pattern
import java.util.stream.Collectors
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlElement, XmlRootElement}
import javax.xml.bind.{JAXBContext, Marshaller}
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.service.PkgDataList
import org.scalatest.flatspec.AnyFlatSpec
import scala.jdk.CollectionConverters._
import scala.annotation.meta.field
@XmlRootElement(name = "List")
@XmlAccessorType(XmlAccessType.FIELD)
class JaxbList2[T] private (@(XmlElement @field)(name = "Item") private val list : util.List[T]) {
def this(s : Seq[T]) {
this(s.toList.asJava)
}
def this() {
this(new util.ArrayList[T]())
}
}
class MarshalTest extends AnyFlatSpec {
"asdfs" should "sdfsd" in {
val context = JAXBContext.newInstance(classOf[PkgId], classOf[PkgDataList])
val mar = context.createMarshaller
mar.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true)
val pkgId2 = new PkgId
pkgId2.name = "linux"
pkgId2.version = "4.6.1"
pkgId2.arch = "x86_64"
val pattern = Pattern.compile(".*\\.pkg\\.tar\\.(zst)")
val pkgDatas = Files.list(Paths.get("/var/cache/pacman/pkg"))
.filter(Files.isRegularFile(_))
.filter((p: Path) => pattern.matcher(p.getFileName.toString).matches)
.limit(10)
.map(Parser.parseFile)
.collect(Collectors.toList[PkgData])
val list = new PkgDataList(pkgDatas)
mar.marshal(list, System.out)
}
}

View File

@@ -0,0 +1,31 @@
package net.woggioni.jpacrepo.sample
import net.woggioni.jpacrepo.persistence.InitialSchemaAction
import net.woggioni.jpacrepo.version.VersionComparator
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import scala.collection.mutable
class ExampleSpec extends AnyFlatSpec with Matchers {
// "A Stack" should "pop values in last-in-first-out order" in {
// val stack = new mutable.Stack[Int]
// stack.push(1)
// stack.push(2)
// stack.pop() should be (2)
// stack.pop() should be (1)
// }
//
// it should "throw NoSuchElementException if an empty stack is popped" in {
// val emptyStack = new mutable.Stack[Int]
// a [NoSuchElementException] should be thrownBy {
// emptyStack.pop()
// }
// }
"sdfgf" should "dfgfd" in {
val initial = InitialSchemaAction("none")
println(initial)
}
}

View File

@@ -1,11 +1,9 @@
package net.woggioni.jpacrepo.service
import javax.enterprise.util.TypeLiteral
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.factory.BeanFactory
import net.woggioni.jpacrepo.persistence.TestPersistenceProducer
import org.jboss.weld.environment.se.Weld
import org.junit.Test
//object WeldContainer {
// private val weld = new Weld
@@ -47,7 +45,6 @@ class PacmanServiceEJBTest {
classOf[BeanFactory],
)
@Test
def test = {
val container = weld.initialize()
try {

View File

@@ -26,4 +26,8 @@ class PacmanWebServiceTest {
// val res = response.getEntity
// response.getStatus
// }
def boh = {
println(System.getProperty("net.woggioni.jpacrepo.configuration.file"))
}
}

View File

@@ -0,0 +1,16 @@
package net.woggioni.jpacrepo.version
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import scala.util.Random
class VersionComparatorSpec extends AnyFlatSpec with Matchers {
"Version sorting" should "work as expected" in {
val vc = new VersionComparator
val originalList = List("1.0", "2.0", "3.0", "5.6.7.arch1-1", "5.6.7.arch3-1", "5.6.7.arch3-2", "20200421.78c0348-1")
val shuffledList = new Random(101325).shuffle(originalList)
val sortedList = shuffledList.sorted(Ordering.comparatorToOrdering(vc))
sortedList should be (originalList)
}
}