woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

conversion from Java to Scala

Browse Source
This commit is contained in:
Walter Oggioni
2019-08-10 13:31:44 +01:00
parent 1cc066d3f0
commit 49734bf6d1
42 changed files with 1320 additions and 1584 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
build.sbt
View File

@@ -10,16 +10,40 @@ version := "2.0"
resolvers += Resolver.mavenLocal
libraryDependencies += "org.tukaani" % "xz" % "1.6"
libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.25"
libraryDependencies += "org.hibernate" % "hibernate-jpamodelgen" % "5.4.2.Final" % Provided
libraryDependencies += "org.apache.commons" % "commons-compress" % "1.14"
libraryDependencies += "org.projectlombok" % "lombok" % "1.18.8" % Provided
libraryDependencies += "javax" % "javaee-api" % "7.0" % Provided
libraryDependencies += "org.hibernate" % "hibernate-jpamodelgen" % "5.3.6.Final" % Provided
libraryDependencies += "junit" % "junit" % "4.12" % Test
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test
libraryDependencies += "com.thoughtworks.xstream" % "xstream" % "1.4.10" % Test
libraryDependencies += "commons-io" % "commons-io" % "2.5" % Test
libraryDependencies += "org.jboss" % "jboss-ejb-client" % "4.0.0.CR5" % Test
libraryDependencies += "org.jboss" % "jboss-ejb-client" % "4.0.18.Final" % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.11.2" % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-core" % "2.11.2" % Test
libraryDependencies += "org.apache.logging.log4j" % "log4j-api" % "2.11.2" % Test
//libraryDependencies += "org.jboss.resteasy" % "resteasy-undertow" % "3.0.4.Final" % Test
//libraryDependencies += "io.undertow" % "undertow-core" % "2.0.20.Final" % Test
//libraryDependencies += "io.undertow" % "undertow-servlet" % "2.0.20.Final" % Test
//libraryDependencies += "org.jboss.weld.servlet" % "weld-servlet-core" % "3.1.1.Final" % Test
libraryDependencies += "org.jboss.weld.se" % "weld-se-core" % "3.1.1.Final" % Test
libraryDependencies += "com.h2database" % "h2" % "1.4.197" % Test
libraryDependencies += "org.hibernate" % "hibernate-core" % "5.3.6.Final" % Test
//libraryDependencies += "io.smallrye" % "smallrye-config" % "1.3.5" % Test
//libraryDependencies += "org.jboss.logging" % "jboss-logging-processor" % "2.2.0.Final" % Test
//libraryDependencies += "org.eclipse.microprofile.config" % "microprofile-config-api" % "1.3" % Test
//libraryDependencies += "org.jboss.spec.javax.ws.rs" % "jboss-jaxrs-api_2.1_spec" % "1.0.2.Final" % Test
//libraryDependencies += "org.reactivestreams" % "reactive-streams" % "1.0.2" % Test
//libraryDependencies += "org.jboss.spec.javax.xml.bind" % "jboss-jaxb-api_2.3_spec" % "1.0.1.Final" % Test
//libraryDependencies += "javax.activation" % "activation" % "1.1.1" % Test
//libraryDependencies += "javax.validation" % "validation-api" % "2.0.1.Final" % Test
//
//libraryDependencies += "org.jboss.resteasy" % "resteasy-cdi" % "4.0.0.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-jaxrs" % "3.0.11.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-client" % "3.0.11.Final" % Test
libraryDependencies += "org.jboss.resteasy" % "resteasy-jackson-provider" % "3.0.11.Final" % Test
@@ -32,6 +56,7 @@ enablePlugins(NimPlugin)
nimCompilerParameters in CompileNim := Seq("-d:release", "-d:serverURL=")
sources in CompileNim := Seq(baseDirectory.value / "nim" / "src" / "jpacrepo.nim")
//fork in Test := true
webappWebInfClasses := true
webappPostProcess := {
@@ -46,17 +71,17 @@ webappPostProcess := {
}
}
lazy val dataSourceJNDI = SettingKey[String]("datasource-jndi", "JNDI name of the application datasource")
lazy val dataBaseAction = SettingKey[String]("database-action",
lazy val datasourceJNDI = SettingKey[String]("datasource-jndi", "JNDI name of the application datasource")
lazy val databaseAction = SettingKey[String]("database-action",
"value of the property \"javax.persistence.schema-generation.database.action\" in the persistence unit")
dataSourceJNDI := "java:/PostgresDS"
dataBaseAction := "none"
datasourceJNDI := "java:/PostgresDS"
databaseAction := "none"
resourceGenerators in Compile += Def.task({
Seq(IO.configureFile(
(sourceDirectory in Compile).value / "resources-template" / "persistence.xml",
(resourceManaged in Compile).value / "META-INF" / "persistence.xml",
Map("dataSourceJNDI" -> dataSourceJNDI.value,
"dataBaseAction" -> dataBaseAction.value)))
Map("dataSourceJNDI" -> datasourceJNDI.value,
"dataBaseAction" -> databaseAction.value)))
}).taskValue

6
nim/src/jpacrepo.nim
View File

@@ -1,6 +1,6 @@
import dom
import htmlutils.tree
import htmlutils.utils
import htmlutils/tree
import htmlutils/utils
import json
import streams
import tables
@@ -25,7 +25,7 @@ type DownloadPanel = ref object
proc newDownloadPanel(parent : Element) : DownloadPanel =
let dp = DownloadPanel()
dp.pkgs = initHashSet[string]()
dp.pkgs = initSet[string]()
htmlTreeAppend(parent):
"div":
classList = ["panel", "panel-default"]

91
src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java
View File

@@ -1,91 +0,0 @@
package com.oggio88.jpacrepo.context;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.service.PacmanServiceView;
import javax.enterprise.context.ApplicationScoped;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
import java.util.Properties;
/**
* Created by walter on 29/03/15.
*/
@ApplicationScoped
public class ApplicationContext
{
public String getRepoFolder()
{
return repoFolder;
}
private Properties systemProperties;
private String repoFolder;
private PacmanServiceView pacmanService;
public boolean invalidateCache = true;
public ApplicationContext(String propertyFile)
{
systemProperties = new Properties();
InputStream input = null;
try
{
input = new FileInputStream(propertyFile);
systemProperties.load(input);
repoFolder = Optional.ofNullable(
System.getProperty("com.oggio88.jpacrepo.RepoFolder")).orElse(
systemProperties.getProperty("RepoFolder"));
}
catch (IOException ex)
{
throw new RuntimeException(ex);
}
finally
{
if (input != null)
{
try
{
input.close();
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
}
public Properties getSystemProperties()
{
return systemProperties;
}
public File getFile(PkgData pkg)
{
return new File(new File(repoFolder), pkg.fileName);
}
public File getFile(String fileName)
{
return new File(new File(repoFolder), fileName);
}
public PacmanServiceView getPacmanService()
{
return pacmanService;
}
public void setPacmanService(PacmanServiceView pacmanService)
{
this.pacmanService = pacmanService;
}
}

26
src/main/java/com/oggio88/jpacrepo/context/ContextProducer.java
View File

@@ -1,26 +0,0 @@
package com.oggio88.jpacrepo.context;
import com.oggio88.jpacrepo.service.PacmanServiceView;
import javax.ejb.EJB;
import javax.enterprise.inject.Produces;
/**
* Created by walter on 04/04/15.
*/
public class ContextProducer
{
@EJB
PacmanServiceView service;
@Produces
@DefaultConfiguration
public ApplicationContext produce()
{
ApplicationContext ctx = new ApplicationContext("/etc/jpacrepo/server.properties");
ctx.setPacmanService(service);
return ctx;
}
}

18
src/main/java/com/oggio88/jpacrepo/context/DefaultConfiguration.java
View File

@@ -1,18 +0,0 @@
package com.oggio88.jpacrepo.context;
import javax.inject.Qualifier;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Created by walter on 04/04/15.
*/
@Qualifier
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.TYPE, ElementType.METHOD, ElementType.PARAMETER})
public @interface DefaultConfiguration
{
}

83
src/main/java/com/oggio88/jpacrepo/model/PkgData.java
View File

@@ -1,83 +0,0 @@
package com.oggio88.jpacrepo.model;
import javax.persistence.*;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.Date;
import java.util.List;
import java.util.Set;
/**
* Created by walter on 22/03/15.
*/
@Entity
@XmlRootElement
@NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id")
@Table(indexes = {@Index(columnList = "md5sum", unique = true), @Index(columnList = "fileName", unique = true)})
public class PkgData
{
@Id
@GeneratedValue
public Integer id;
@ManyToOne(cascade = CascadeType.PERSIST)
public PkgName name;
public String base;
public String version;
public String description;
public String url;
@Temporal(TemporalType.TIMESTAMP)
public Date buildDate;
public String packager;
public Long size;
public String arch;
public String license;
public String md5sum;
public String fileName;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> replaces;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> conflict;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> provides;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> depend;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> optdepend;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> makedepend;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> makeopkgopt;
@ElementCollection(fetch = FetchType.EAGER)
public Set<String> backup;
@Temporal(TemporalType.TIMESTAMP)
public Date updTimestamp;
@PreUpdate
@PrePersist
private void writeTimestamp()
{
updTimestamp = new Date();
}
}

48
src/main/java/com/oggio88/jpacrepo/model/PkgList.java
View File

@@ -1,48 +0,0 @@
package com.oggio88.jpacrepo.model;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* Created by walter on 29/03/15.
*/
@XmlRootElement
@XmlSeeAlso(PkgData.class)
public class PkgList extends ArrayList<PkgData>
{
// ======================================
// = Constructors =
// ======================================
public PkgList()
{
super();
}
public PkgList(Collection<? extends PkgData> c)
{
super(c);
}
// ======================================
// = Getters & Setters =
// ======================================
@XmlElement(name = "PkgData")
public List<PkgData> getPackages()
{
return this;
}
public void setPackages(List<PkgData> pkgs)
{
this.clear();
this.addAll(pkgs);
}
}

24
src/main/java/com/oggio88/jpacrepo/model/PkgName.java
View File

@@ -1,24 +0,0 @@
package com.oggio88.jpacrepo.model;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.xml.bind.annotation.XmlRootElement;
/**
* Created by walter on 22/03/15.
*/
@Entity
@XmlRootElement
public class PkgName
{
@Id
public String id;
public PkgName(){};
public PkgName(String name)
{
id = name;
}
}

47
src/main/java/com/oggio88/jpacrepo/model/StringList.java
View File

@@ -1,47 +0,0 @@
package com.oggio88.jpacrepo.model;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* Created by walter on 29/03/15.
*/
@XmlRootElement
public class StringList extends ArrayList<String>
{
// ======================================
// = Constructors =
// ======================================
public StringList()
{
super();
}
public StringList(Collection<? extends String> c)
{
super(c);
}
// ======================================
// = Getters & Setters =
// ======================================
@XmlElement(name = "string")
public List<String> getPackages()
{
return this;
}
public void setPackages(List<String> pkgs)
{
this.clear();
this.addAll(pkgs);
}
}

95
src/main/java/com/oggio88/jpacrepo/pacbase/Hasher.java
View File

@@ -1,95 +0,0 @@
package com.oggio88.jpacrepo.pacbase;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/**
* Created by walter on 05/04/15.
*/
public class Hasher
{
static private final MessageDigest md5;
private MessageDigest md;
static
{
try
{
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e)
{
throw new RuntimeException(e);
}
}
public Hasher(String algorithm)
{
try
{
md = MessageDigest.getInstance(algorithm);
}
catch (NoSuchAlgorithmException e)
{
throw new RuntimeException(e);
}
}
public static String computeMD5(InputStream is) throws IOException
{
byte[] buffer = new byte[100000];
while (is.available() > 0)
{
int read = is.read(buffer, 0, 100000);
md5.update(buffer, 0, read);
}
is.close();
return bytesToHex(md5.digest());
}
public String getHashString(InputStream is) throws IOException
{
byte[] buffer = new byte[100000];
while (is.available() > 0)
{
int read = is.read(buffer, 0, 100000);
md.update(buffer, 0, read);
}
is.close();
return bytesToHex(md.digest());
}
final private static char[] hexArray = "0123456789ABCDEF".toCharArray();
public static String bytesToHex(byte[] bytes)
{
char[] hexChars = new char[bytes.length * 2];
for (int j = 0; j < bytes.length; j++)
{
int v = bytes[j] & 0xFF;
hexChars[j * 2] = hexArray[v >>> 4];
hexChars[j * 2 + 1] = hexArray[v & 0x0F];
}
return new String(hexChars);
}
static MessageDigest getMd5()
{
try
{
return MessageDigest.getInstance("MD5");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public static MD5InputStream createMD5InputStream(InputStream is)
{
return new MD5InputStream(is);
}
}

21
src/main/java/com/oggio88/jpacrepo/pacbase/MD5InputStream.java
View File

@@ -1,21 +0,0 @@
package com.oggio88.jpacrepo.pacbase;
import java.io.InputStream;
import java.security.DigestInputStream;
/**
* Created by walter on 25/04/15.
*/
public class MD5InputStream extends DigestInputStream
{
MD5InputStream(InputStream is)
{
super(is, Hasher.getMd5());
}
public String digest()
{
byte[] digest = getMessageDigest().digest();
return Hasher.bytesToHex(digest);
}
}

133
src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java
View File

@@ -1,133 +0,0 @@
package com.oggio88.jpacrepo.pacbase;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgName;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.nio.charset.Charset;
import java.util.*;
/**
* Created by walter on 22/03/15.
*/
public class Parser
{
public static PkgData parseFile(File file) throws Exception
{
XZCompressorInputStream xzcis;
TarArchiveInputStream tais;
ArchiveEntry ae;
Hasher hasher = new Hasher("MD5");
xzcis = new XZCompressorInputStream(new FileInputStream(file));
tais = new TarArchiveInputStream(xzcis);
while ((ae = tais.getNextEntry()) != null)
{
if (ae.getName().equals(".PKGINFO"))
{
Map<String, List<String>> propMap = new HashMap<>();
byte[] buffer = new byte[(int) tais.getCurrentEntry().getSize()];
tais.read(buffer);
String info = new String(buffer, Charset.forName("UTF8"));
for (String line : info.split("\n"))
{
if (line.startsWith("#") || line.trim().length() == 0)
{
continue;
}
else
{
int equals = line.indexOf("=");
if (equals < 0)
{
throw new RuntimeException("Error parsing .PKGINFO file");
}
else
{
String key = line.substring(0, equals).trim();
if (propMap.get(key) == null)
{
propMap.put(key, new ArrayList<>());
}
propMap.get(key).add(line.substring(equals + 1, line.length()).trim());
}
}
}
tais.close();
xzcis.close();
PkgData data = new PkgData();
for (String key : propMap.keySet())
{
switch (key)
{
case "size":
data.size = Long.valueOf(propMap.get(key).get(0));
break;
case "arch":
data.arch = propMap.get(key).get(0);
break;
case "replaces":
data.replaces = new HashSet(propMap.get(key));
break;
case "packager":
data.packager = propMap.get(key).get(0);
break;
case "url":
data.url = propMap.get(key).get(0);
break;
case "pkgname":
data.name = new PkgName(propMap.get(key).get(0));
break;
case "builddate":
data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0)) * 1000);
break;
case "license":
data.license = propMap.get(key).get(0);
break;
case "pkgver":
data.version = propMap.get(key).get(0);
break;
case "pkgdesc":
data.description = propMap.get(key).get(0);
break;
case "provides":
data.provides = new HashSet(propMap.get(key));
break;
case "conflict":
data.conflict = new HashSet(propMap.get(key));
break;
case "backup":
data.backup = new HashSet(propMap.get(key));
break;
case "optdepend":
data.optdepend = new HashSet(propMap.get(key));
break;
case "depend":
data.depend = new HashSet(propMap.get(key));
break;
case "makedepend":
data.makedepend = new HashSet(propMap.get(key));
break;
case "makepkgopt":
data.makeopkgopt = new HashSet(propMap.get(key));
break;
case "pkgbase":
data.base = propMap.get(key).get(0);
break;
}
}
data.md5sum = hasher.getHashString(new FileInputStream(file));
data.fileName = file.getName();
return data;
}
}
tais.close();
xzcis.close();
return null;
}
}

62
src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java
View File

@@ -1,62 +0,0 @@
package com.oggio88.jpacrepo.persistence;
import java.util.ArrayList;
import java.util.List;
/**
* Created by walter on 29/03/15.
*/
public class QueryEngine
{
private String entityName;
private String query;
private List<String> where;
public QueryEngine(Class<?> cls)
{
query = String.format("SELECT e FROM %s e", cls.getSimpleName());
this.entityName = cls.getSimpleName();
where = new ArrayList<>();
}
public QueryEngine(String entityName)
{
query = String.format("SELECT e FROM %s e", entityName);
where = new ArrayList<>();
}
public QueryEngine select(String... fields)
{
String[] strarr = new String[fields.length];
for (int i = 0; i < fields.length; i++)
{
strarr[i] = "e." + fields[i];
}
query = "SELECT " + String.join(",", strarr) + " FROM " + entityName + " e";
return this;
}
public QueryEngine select()
{
query = String.format("SELECT e FROM %s e", entityName);
return this;
}
public QueryEngine where(String field, String operator, String value)
{
where.add(String.format("e.%s %s '%s'", field, operator, value));
return this;
}
public String build()
{
if (where.isEmpty())
{
return query;
}
else
{
return query + " WHERE " + String.join(" AND ", where);
}
}
}

39
src/main/java/com/oggio88/jpacrepo/service/ApplicationConfig.java
View File

@@ -1,39 +0,0 @@
package com.oggio88.jpacrepo.service;
import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@ApplicationPath("rest")
public class ApplicationConfig extends Application
{
// ======================================
// = Attributes =
// ======================================
private final Set<Class<?>> classes;
// ======================================
// = Constructors =
// ======================================
public ApplicationConfig()
{
HashSet<Class<?>> c = new HashSet<>();
c.add(PacmanWebService.class);
classes = Collections.unmodifiableSet(c);
}
// ======================================
// = Getters & Setters =
// ======================================
@Override
public Set<Class<?>> getClasses() {
return classes;
}
}

295
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -1,295 +0,0 @@
package com.oggio88.jpacrepo.service;
import com.oggio88.jpacrepo.context.ApplicationContext;
import com.oggio88.jpacrepo.context.DefaultConfiguration;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.pacbase.Parser;
import lombok.SneakyThrows;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.ejb.*;
import javax.enterprise.concurrent.ManagedThreadFactory;
import javax.inject.Inject;
import javax.persistence.*;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.transaction.*;
import javax.transaction.RollbackException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
@Startup
@Singleton
@Lock(LockType.READ)
@TransactionManagement(TransactionManagementType.BEAN)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local(PacmanServiceView.class)
@Remote(PacmanServiceRemote.class)
public class PacmanServiceEJB implements PacmanServiceView {
@PersistenceUnit(unitName = "jpacrepo_pu")
private EntityManagerFactory emf;
@Inject
@DefaultConfiguration
private ApplicationContext ctx;
@Resource
private UserTransaction ut;
private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName());
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
@SneakyThrows
@Asynchronous
@Schedule(hour = "3", minute = "00", persistent = false)
public void syncDB() {
EntityManager em = emf.createEntityManager();
Set<String> knownPkg;
logger.log(Level.INFO, "Starting repository cleanup");
knownPkg = new HashSet<>();
//Elimina i pacchetti sul DB che non esistono più nel filesystem
logger.info("Searching for packages that are no more in the filesystem");
ut.setTransactionTimeout(86400);
ut.begin();
List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList();
logger.info("Got list of filenames from db");
for(String fileName : listaDB) {
File file = ctx.getFile(fileName);
if(!file.exists()) {
logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName()));
PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName",
PkgData.class).setParameter("fileName", file.getName()).getSingleResult();
em.remove(pkg);
} else {
knownPkg.add(fileName);
}
}
logger.info("Searching for new packages or packages that were modified after being added to the database");
List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
Parser parser = new Parser();
for(File file : pkgfiles) {
boolean update = !knownPkg.contains(file.getName());
if(!update) {
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getName());
Date result = query.getSingleResult();
if(file.lastModified() > result.getTime()) {
update = true;
}
}
if(update) {
try {
parseFile(em, parser, file);
} catch(Exception e) {
logger.log(Level.SEVERE, String.format("Error parsing %s", file.getAbsolutePath()), e);
}
}
}
logger.log(Level.INFO, "Removing obsolete packages");
deleteOld(em);
ut.commit();
logger.info("Repository cleanup completed successfully");
ctx.invalidateCache = true;
}
private void parseFile(EntityManager em, Parser p, File file) throws Exception {
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum);
try {
hquery.getSingleResult();
return;
} catch(NoResultException e) {
}
try {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", file.getName());
PkgData savedFile = fquery.getSingleResult();
em.remove(savedFile);
em.flush();
} catch(NoResultException e) {
}
nquery.setParameter("name", data.name.id);
try {
PkgName savedName = nquery.getSingleResult();
data.name = savedName;
} catch(NoResultException e) {
}
em.persist(data);
logger.log(Level.INFO, String.format("Persisting package %s", file.getName()));
}
@Override
public void deletePackage(String filename) throws Exception {
EntityManager em = emf.createEntityManager();
ut.begin();
deletePackage(em, filename);
ut.commit();
logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
}
public void deletePackage(EntityManager em, String filename) throws Exception {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList();
if(savedFiles.size() == 0) {
ut.rollback();
throw new RuntimeException(String.format("Package with name %s not found", filename));
}
PkgData pkg = fquery.getResultList().get(0);
try {
Files.delete(ctx.getFile(pkg).toPath());
} catch(IOException e) {
ut.rollback();
throw new RuntimeException(e);
}
em.remove(pkg);
}
private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" +
"(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)";
private void deleteOld(EntityManager em) {
TypedQuery<String> query = em.createQuery(deleteQuery, String.class);
Calendar cutoff = Calendar.getInstance();
cutoff.add(Calendar.YEAR, -2);
query.setParameter("cutoff", cutoff.getTime());
query.setParameter("minVersions", (long) 2);
List<String> list = query.getResultList();
list.forEach((el) ->
{
try {
deletePackage(el);
} catch(Exception e) {
throw new RuntimeException(e);
}
});
}
@Override
public long countResults(String name, String version, String arch) {
EntityManager em = emf.createEntityManager();
CriteriaBuilder builder;
CriteriaQuery<Long> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(Long.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if(name != null && !name.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if(version != null && !version.isEmpty()) {
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(arch != null && !arch.isEmpty()) {
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(finalPredicate != null) {
criteriaQuery.select(builder.count(entity)).where(finalPredicate);
} else {
criteriaQuery.select(builder.count(entity));
}
return em.createQuery(criteriaQuery).getSingleResult();
}
@Override
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls) {
EntityManager em = emf.createEntityManager();
CriteriaBuilder builder = em.getCriteriaBuilder();
CriteriaQuery<T> criteriaQuery = builder.createQuery(cls);
Root<PkgData> entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
String key = equalityConditions.get("name");
if(key != null && !key.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), key);
finalPredicate = p;
}
key = equalityConditions.get("version");
if(key != null && !key.isEmpty()) {
p = builder.equal(entity.get("version"), key);
finalPredicate = p;
}
key = equalityConditions.get("arch");
if(key != null && !key.isEmpty()) {
p = builder.equal(entity.get("arch"), key);
finalPredicate = p;
}
if(finalPredicate != null) {
criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate);
} else {
criteriaQuery.select(entity.get(property)).distinct(true);
}
return em.createQuery(criteriaQuery).getResultList();
}
@Override
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize) {
EntityManager em = emf.createEntityManager();
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(PkgData.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if(name != null && !name.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if(version != null && !version.isEmpty()) {
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(arch != null && !arch.isEmpty()) {
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(finalPredicate != null) {
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
} else {
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if(pageNumber >= 0) {
query.setFirstResult(pageNumber * pageSize);
}
if(pageSize > 0) {
query.setMaxResults(pageSize);
}
return query.getResultList();
}
}

14
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java
View File

@@ -1,14 +0,0 @@
package com.oggio88.jpacrepo.service;
import javax.ejb.Remote;
/**
* Created by walter on 08/07/15.
*/
@Remote
public interface PacmanServiceRemote
{
public void syncDB();
public void deletePackage(String filename) throws Exception;
}

22
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceView.java
View File

@@ -1,22 +0,0 @@
package com.oggio88.jpacrepo.service;
import com.oggio88.jpacrepo.model.PkgData;
import javax.ejb.Local;
import java.util.List;
import java.util.Map;
/**
* Created by walter on 28/03/15.
*/
@Local
public interface PacmanServiceView extends PacmanServiceRemote
{
long countResults(String name, String version, String arch);
List<PkgData> searchPackage(String name, String version, String arch, int page, int pageSize);
<T> List<T> listProperty(String property, Map<String,String> equalityConditions, Class<T> cls);
}

516
src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java
View File

@@ -1,516 +0,0 @@
package com.oggio88.jpacrepo.service;
import com.oggio88.jpacrepo.context.ApplicationContext;
import com.oggio88.jpacrepo.context.DefaultConfiguration;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgList;
import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.model.StringList;
import com.oggio88.jpacrepo.pacbase.Parser;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import javax.annotation.Resource;
import javax.ejb.Singleton;
import javax.ejb.TransactionManagement;
import javax.ejb.TransactionManagementType;
import javax.inject.Inject;
import javax.persistence.*;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.transaction.Status;
import javax.transaction.UserTransaction;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.*;
import java.net.URI;
import java.nio.file.Files;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
@XmlRootElement
class PkgTuple
{
String md5sum;
public String filename;
public long size;
}
@Singleton
@Path("/pkg")
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@TransactionManagement(TransactionManagementType.BEAN)
public class PacmanWebService
{
private SortedMap<String, SortedMap<String, SortedMap<String, PkgTuple>>> cachedMap = null;
@PersistenceContext(unitName = "jpacrepo_pu")
private EntityManager em;
@Resource
private UserTransaction ut;
@Context
private UriInfo uriInfo;
private final Logger log = Logger.getLogger(PacmanWebService.class.getName());
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String fileNameQuery = "SELECT pdata FROM PkgData pdata WHERE fileName = :fileName";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
@Inject
@DefaultConfiguration
private ApplicationContext ctx;
private SortedMap<String, SortedMap<String, SortedMap<String, PkgTuple>>> getCachedMap()
{
SortedMap<String, SortedMap<String, SortedMap<String, PkgTuple>>> result;
if (ctx.invalidateCache)
{
result = new TreeMap<>();
TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName, pkg.size, pkg.md5sum FROM PkgData pkg", Object[].class);
for (Object[] pkg : query.getResultList())
{
String name = (String) pkg[0];
String version = (String) pkg[1];
String arch = (String) pkg[2];
String filename = (String) pkg[3];
long size = (long) pkg[4];
String md5sum = (String) pkg[5];
result.putIfAbsent(name, new TreeMap<>());
SortedMap<String, SortedMap<String, PkgTuple>> map = result.get(name);
map.putIfAbsent(version, new TreeMap<>());
SortedMap<String, PkgTuple> map2 = map.get(version);
PkgTuple tuple = new PkgTuple();
tuple.filename = filename;
tuple.size = size;
tuple.md5sum = md5sum;
map2.putIfAbsent(arch, tuple);
}
cachedMap = result;
ctx.invalidateCache = false;
}
else
{
result = cachedMap;
}
return result;
}
@GET
@Path("searchByName/{name}")
public Response searchByName(@PathParam("name") String name)
{
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name);
return Response.ok(em.createQuery(query, String.class).getResultList()).build();
}
@GET
@Path("searchByHash/{md5sum}")
public Response searchByHash(@PathParam("md5sum") String md5sum)
{
return getPackageByHash(md5sum);
}
@GET
@Path("list/{name}")
public Response getPackage(@PathParam("name") String name)
{
TypedQuery query = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", String.class);
query.setParameter("name", name);
return Response.ok(query.getResultList()).build();
}
@GET
@Path("list/{name}/{version}")
public Response getPackage(@PathParam("name") String name, @PathParam("version") String version)
{
TypedQuery query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", String.class);
query.setParameter("name", name);
query.setParameter("version", version);
return Response.ok(query.getResultList()).build();
}
@GET
@Path("list/{name}/{version}/{arch}")
public Response getPackage(@PathParam("name") String name, @PathParam("version") String version, @PathParam("arch") String arch)
{
TypedQuery<PkgData> query = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " +
"pkg.name.id = :name AND " +
"pkg.version = :version AND " +
"pkg.arch = :arch " +
"ORDER BY pkg.arch", PkgData.class);
query.setParameter("name", name);
query.setParameter("version", version);
query.setParameter("arch", arch);
return Response.ok(query.getSingleResult()).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("map")
public Response getPackageMap(@Context Request request)
{
CacheControl cc = new CacheControl();
cc.setMaxAge(86400);
cc.setMustRevalidate(true);
cc.setNoCache(true);
SortedMap<String, SortedMap<String, SortedMap<String, PkgTuple>>> result = getCachedMap();
EntityTag etag = new EntityTag(Integer.toString(result.hashCode()));
Response.ResponseBuilder builder = request.evaluatePreconditions(etag);
if (builder == null)
{
builder = Response.ok(result);
builder.tag(etag);
}
builder.cacheControl(cc);
return builder.build();
}
@GET
@Path("hashes")
public Response getHashes()
{
Query query = em.createQuery("SELECT p.md5sum FROM PkgData p");
List<String> hashes = query.getResultList();
StringList hl = new StringList(hashes);
return Response.ok(hl).build();
}
@GET
@Path("files")
public Response getFiles()
{
Query query = em.createQuery("SELECT p.fileName FROM PkgData p");
List<String> hashes = query.getResultList();
StringList hl = new StringList(hashes);
return Response.ok(hl).build();
}
private Response getPackageByHash(String md5sum)
{
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
if (md5sum != null) hquery.setParameter("md5sum", md5sum);
return manageQueryResult(hquery.getResultList(), true);
}
private Response getPackageByFileName(String file)
{
TypedQuery<PkgData> fnquery = em.createQuery(fileNameQuery, PkgData.class);
fnquery.setParameter("fileName", file);
return manageQueryResult(fnquery.getResultList(), true);
}
@GET
@Path("filesize/{filename}")
public Response getFileSize(@PathParam("filename") String fileName, @Context Request request)
{
CacheControl cc = new CacheControl();
cc.setMaxAge(86400);
cc.setMustRevalidate(true);
cc.setNoCache(true);
EntityTag etag = new EntityTag(Integer.toString(getCachedMap().hashCode()));
Response.ResponseBuilder builder = request.evaluatePreconditions(etag);
if (builder == null)
{
File res = ctx.getFile(fileName);
if (!res.exists()) throw new NotFoundException(String.format("File '%s' was not found", fileName));
builder = Response.ok(res.length());
builder.tag(etag);
}
builder.cacheControl(cc);
return builder.build();
}
@GET
@Path("download/{filename}")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
public Response downloadPackage(@PathParam("filename") String fileName)
{
TypedQuery<PkgData> fnquery = em.createQuery(fileNameQuery, PkgData.class);
fnquery.setParameter("fileName", fileName);
try
{
PkgData pkg = fnquery.getSingleResult();
StreamingOutput stream = (OutputStream output) ->
{
FileInputStream input = new FileInputStream(ctx.getFile(pkg));
try
{
byte[] bytes = new byte[1024];
while (true)
{
int size = input.read(bytes);
if (size < 0) break;
else output.write(bytes, 0, size);
}
output.close();
}
catch (Exception e)
{
throw new WebApplicationException(e);
}
finally
{
input.close();
}
};
return Response.ok(stream).header("Content-Length", ctx.getFile(pkg).length()).build();
}
catch (NonUniqueResultException e)
{
throw new InternalServerErrorException();
}
catch (NoResultException e)
{
throw new NotFoundException();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
@POST
@Path("/doYouWantAny")
@Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public Response doYouWantAny(List<String> filenames)
{
Set<String> result = new HashSet<>(filenames);
if(!result.isEmpty())
{
TypedQuery<String> query = em.createQuery("SELECT pkg.fileName from PkgData pkg WHERE pkg.fileName in :filenames", String.class);
query.setParameter("filenames", filenames);
Set<String> toBeRemoved = new HashSet(query.getResultList());
result.removeAll(toBeRemoved);
}
return Response.ok(result).build();
}
@POST
@Path("/upload")
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
public Response createPackage(InputStream input, @MatrixParam("filename") String filename) throws Exception
{
if (filename == null)
throw new BadRequestException();
File file = File.createTempFile(filename, "tmp", new File(ctx.getRepoFolder()));
TypedQuery<PkgData> fquery = em.createQuery(fileNameQuery, PkgData.class);
fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList();
if (savedFiles.size() > 0)
{
return Response.notModified().build();
}
else
{
try
{
FileOutputStream fos = new FileOutputStream(file);
byte[] buffer = new byte[4096];
while (true)
{
int size = input.read(buffer);
if (size < 0) break;
else fos.write(buffer, 0, size);
}
fos.close();
ut.begin();
PkgData pkg = Parser.parseFile(file);
pkg.fileName = filename;
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
nquery.setParameter("name", pkg.name.id);
List<PkgName> savedName = nquery.getResultList();
if (savedName.size() > 0)
{
pkg.name = savedName.get(0);
}
em.persist(pkg);
log.log(Level.INFO, String.format("Persisting package %s", pkg.fileName));
URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.fileName).build();
ut.commit();
Files.move(file.toPath(), new File(ctx.getRepoFolder(), filename).toPath(), ATOMIC_MOVE);
ctx.invalidateCache = true;
cachedMap = null;
return Response.created(pkgUri).build();
}
catch (Exception e)
{
e.printStackTrace();
Files.delete(file.toPath());
if (ut.getStatus() != Status.STATUS_NO_TRANSACTION)
{
ut.rollback();
}
throw e;
}
}
}
@GET
@Path("/search")
public List<PkgData> searchPackage(
@QueryParam("name") String name,
@QueryParam("version") String version,
@QueryParam("arch") String arch,
@QueryParam("page") int pageNumber,
@QueryParam("pageSize") int pageSize,
@QueryParam("fileName") String fileName
)
{
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(PkgData.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (fileName != null && !fileName.isEmpty())
{
p = builder.equal(entity.get("fileName"), fileName);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
else
{
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0)
{
query.setFirstResult(pageNumber * pageSize);
}
if (pageSize > 0)
{
query.setMaxResults(pageSize);
}
return query.getResultList();
}
@OPTIONS
@Path("/downloadTar")
@Produces("text/plain; charset=UTF-8")
public Response options()
{
return Response.ok("POST, OPTIONS").build();
}
@POST
@Path("/downloadTar")
@Produces("application/x-tar")
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
public Response downloadTar(@FormParam("pkgs") String formData)
{
String[] files = formData.split(" ");
for (String fname : files)
{
if (!ctx.getFile(fname).exists())
throw new NotFoundException(String.format("Package file '%s' does not exist", fname));
}
StreamingOutput stream = new StreamingOutput()
{
@Override
public void write(OutputStream output) throws IOException, WebApplicationException
{
TarArchiveOutputStream taos = new TarArchiveOutputStream(output);
try
{
for (String fname : files)
{
File file = ctx.getFile(fname);
FileInputStream input = new FileInputStream(file);
TarArchiveEntry entry = new TarArchiveEntry(fname);
entry.setSize(file.length());
taos.putArchiveEntry(entry);
byte[] bytes = new byte[1024];
while (true)
{
int size = input.read(bytes);
if (size < 0) break;
else taos.write(bytes, 0, size);
}
taos.closeArchiveEntry();
}
taos.close();
}
catch (Exception e)
{
throw new WebApplicationException(e);
}
finally
{
taos.close();
}
}
};
return Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build();
}
private Response manageQueryResult(List<PkgData> list)
{
return manageQueryResult(list, false);
}
private Response manageQueryResult(List<PkgData> list, boolean singleResult)
{
PkgList pkgList = new PkgList(list);
if (pkgList.size() == 0)
{
throw new NotFoundException();
}
else if (singleResult)
{
if (pkgList.size() == 1)
{
return Response.ok(pkgList.get(0)).build();
}
else
{
throw new NonUniqueResultException("The returned list does not contain a single element");
}
}
else
{
return Response.ok(pkgList).build();
}
}
}

161
src/main/java/net/woggioni/jpacrepo/persistence/QueryEngine.java Normal file
View File

@@ -0,0 +1,161 @@
package net.woggioni.jpacrepo.persistence;
import net.woggioni.jpacrepo.pacbase.PkgData;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import java.util.ArrayList;
import java.util.List;
/**
* Created by walter on 29/03/15.
*/
public class QueryEngine
{
private String entityName;
private String query;
private List<String> where;
public QueryEngine(Class<?> cls)
{
query = String.format("SELECT e FROM %s e", cls.getSimpleName());
this.entityName = cls.getSimpleName();
where = new ArrayList<>();
}
public QueryEngine(String entityName)
{
query = String.format("SELECT e FROM %s e", entityName);
where = new ArrayList<>();
}
public QueryEngine select(String... fields)
{
String[] strarr = new String[fields.length];
for (int i = 0; i < fields.length; i++)
{
strarr[i] = "e." + fields[i];
}
query = "SELECT " + String.join(",", strarr) + " FROM " + entityName + " e";
return this;
}
public QueryEngine select()
{
query = String.format("SELECT e FROM %s e", entityName);
return this;
}
public QueryEngine where(String field, String operator, String value)
{
where.add(String.format("e.%s %s '%s'", field, operator, value));
return this;
}
public String build()
{
if (where.isEmpty())
{
return query;
}
else
{
return query + " WHERE " + String.join(" AND ", where);
}
}
public static List<PkgData> searchPackage(
EntityManager em, String name, String version, String arch, int pageNumber, int pageSize, String fileName)
{
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(PkgData.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (fileName != null && !fileName.isEmpty())
{
p = builder.equal(entity.get("fileName"), fileName);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
else
{
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0)
{
query.setFirstResult(pageNumber * pageSize);
}
if (pageSize > 0)
{
query.setMaxResults(pageSize);
}
return query.getResultList();
}
public static long countResults(EntityManager em, String name, String version, String arch)
{
CriteriaBuilder builder;
CriteriaQuery<Long> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(Long.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
criteriaQuery.select(builder.count(entity)).where(finalPredicate);
}
else
{
criteriaQuery.select(builder.count(entity));
}
return em.createQuery(criteriaQuery).getSingleResult();
}
}

2
src/main/java/com/oggio88/jpacrepo/servlet/AbstractFileServlet.java → src/main/java/net/woggioni/jpacrepo/servlet/AbstractFileServlet.java
View File

@@ -1,4 +1,4 @@
package com.oggio88.jpacrepo.servlet;
package net.woggioni.jpacrepo.servlet;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;

15
src/main/java/com/oggio88/jpacrepo/servlet/FileServlet.java → src/main/java/net/woggioni/jpacrepo/servlet/FileServlet.java
View File

@@ -1,27 +1,20 @@
package com.oggio88.jpacrepo.servlet;
package net.woggioni.jpacrepo.servlet;
import com.oggio88.jpacrepo.context.ApplicationContext;
import com.oggio88.jpacrepo.context.DefaultConfiguration;
import net.woggioni.jpacrepo.context.ApplicationContext;
import javax.inject.Inject;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
/**
* Created by walter on 29/07/15.
*/
@WebServlet("/archive/*")
public class FileServlet extends AbstractFileServlet
{
private String root;
@Inject
public FileServlet(@DefaultConfiguration ApplicationContext ctx){
root = ctx.getRepoFolder();
public FileServlet(ApplicationContext ctx){
root = ctx.repoFolder();
}
@Override

2
src/main/resources-template/persistence.xml
View File

@@ -7,6 +7,7 @@
<persistence-unit name="jpacrepo_pu" transaction-type="JTA">
<jta-data-source>${dataSourceJNDI}</jta-data-source>
<properties>
<property name="org.jboss.logging.provider" value="log4j2"/>
<property name="javax.persistence.schema-generation.database.action" value="${dataBaseAction}"/>
<property name="javax.persistence.schema-generation.create-source" value="script-then-metadata"/>
<property name="javax.persistence.schema-generation.create-script-source"
@@ -14,7 +15,6 @@
<property name="javax.persistence.schema-generation.drop-source" value="metadata-then-script"/>
<property name="javax.persistence.schema-generation.drop-script-source"
value="META-INF/sql/DropSchema.sql"/>
<property name="eclipselink.logging.level" value="INFO"/>
<property name="hibernate.default_schema" value="jpacrepo"/>
</properties>
</persistence-unit>

1
src/main/resources/META-INF/sql/DropSchema.sql Normal file
View File

@@ -0,0 +1 @@
DROP SCHEMA IF EXISTS jpacrepo;

15
src/main/scala/net/woggioni/jpacrepo/config/ApplicationConfig.scala Normal file
View File

@@ -0,0 +1,15 @@
package net.woggioni.jpacrepo.config
import javax.ws.rs.ApplicationPath
import javax.ws.rs.core.Application
import net.woggioni.jpacrepo.service.PacmanWebService
import scala.collection.JavaConverters._
@ApplicationPath("rest")
class ApplicationConfig() extends Application {
val classes : Set[Class[_]] = Set(classOf[PacmanWebService])
override def getClasses = classes.asJava
}

42
src/main/scala/net/woggioni/jpacrepo/context/ApplicatioContext.scala Normal file
View File

@@ -0,0 +1,42 @@
package net.woggioni.jpacrepo.context
import java.io.{File, FileInputStream}
import java.util.Properties
import javax.enterprise.context.ApplicationScoped
import net.woggioni.jpacrepo.pacbase.PkgData
import net.woggioni.jpacrepo.service.PacmanServiceView
@ApplicationScoped
class ApplicationContext(val propertyFile: String) {
val systemProperties = {
val result = new Properties()
val input = new FileInputStream(propertyFile)
try {
result.load(input)
} finally {
input.close()
}
result
}
val repoFolder = System.getProperty("net.woggioni.jpacrepo.RepoFolder") match {
case null => systemProperties.getProperty("RepoFolder")
case path: String => path
}
private var pacmanService: PacmanServiceView = null
var invalidateCache = true
def getFile(pkg: PkgData) = new File(new File(repoFolder), pkg.fileName)
def getFile(fileName: String) = new File(new File(repoFolder), fileName)
def getPacmanService: PacmanServiceView = pacmanService
def setPacmanService(pacmanService: PacmanServiceView): Unit = {
this.pacmanService = pacmanService
}
}

35
src/main/scala/net/woggioni/jpacrepo/factory/BeanFactory.scala Normal file
View File

@@ -0,0 +1,35 @@
package net.woggioni.jpacrepo.factory
import javax.ejb.EJB
import javax.enterprise.inject.Produces
import javax.enterprise.inject.spi.InjectionPoint
import javax.persistence.{EntityManagerFactory, PersistenceUnit}
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.service.PacmanServiceView
import org.slf4j.LoggerFactory
class PersistenceUnitFactory {
@PersistenceUnit(unitName = "jpacrepo_pu")
private var emf : EntityManagerFactory = _
@Produces
private def createEntityManagerFactory = emf
}
class BeanFactory {
@EJB
private var service : PacmanServiceView = _
@Produces
def produce: ApplicationContext = {
val ctx = new ApplicationContext("/etc/jpacrepo/server.properties")
ctx.setPacmanService(service)
ctx
}
@Produces
private def createLogger(injectionPoint: InjectionPoint) =
LoggerFactory.getLogger(injectionPoint.getMember.getDeclaringClass.getName)
}

46
src/main/scala/net/woggioni/jpacrepo/model/Hasher.scala Normal file
View File

@@ -0,0 +1,46 @@
package net.woggioni.jpacrepo.model
import java.io.InputStream
import java.security.MessageDigest
object Hasher {
private val md5 = MessageDigest.getInstance("MD5")
private def getHash(md : MessageDigest, is: InputStream): Array[Byte] = {
val buffer = new Array[Byte](4096)
var read = 0
while ( {
read = is.read(buffer, 0, buffer.length)
read >= 0
}) {
md.update(buffer, 0, read)
}
md.digest
}
def computeMD5(is: InputStream): String = Hasher.bytesToHex(getHash(md5, is))
private val hexArray = "0123456789ABCDEF".toCharArray
def bytesToHex(bytes: Array[Byte]): String = {
val hexChars = new Array[Char](bytes.length * 2)
var j = 0
while (j < bytes.length) {
val v = bytes(j) & 0xFF
hexChars(j * 2) = hexArray(v >>> 4)
hexChars(j * 2 + 1) = hexArray(v & 0x0F)
j += 1
}
new String(hexChars)
}
}
class Hasher(val algorithm: String) {
private val md = MessageDigest.getInstance(algorithm)
def getHash(is : InputStream) = Hasher.getHash(md, is)
def getHashString(is: InputStream): String = Hasher.bytesToHex(getHash(is))
}

11
src/main/scala/net/woggioni/jpacrepo/model/MD5InputStream.scala Normal file
View File

@@ -0,0 +1,11 @@
package net.woggioni.jpacrepo.model
import java.io.InputStream
import java.security.{DigestInputStream, MessageDigest}
class MD5InputStream(val is: InputStream) extends DigestInputStream(is, MessageDigest.getInstance("md5")) {
def digest(): String = {
val digest = getMessageDigest.digest
Hasher.bytesToHex(digest)
}
}

105
src/main/scala/net/woggioni/jpacrepo/model/Parser.scala Normal file
View File

@@ -0,0 +1,105 @@
package net.woggioni.jpacrepo.model
import java.io.{BufferedInputStream, File, FileInputStream}
import java.util.Date
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import org.apache.commons.compress.archivers.ArchiveEntry
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream
import scala.collection.JavaConverters._
import scala.io.Source
object Parser {
def parseFile(file: File): PkgData = {
val hasher = new Hasher("MD5")
val is = new TarArchiveInputStream(
new XZCompressorInputStream(
new BufferedInputStream(
new FileInputStream(file))))
try {
var archiveEntry = is.getNextEntry
while (archiveEntry != null) {
if (archiveEntry.getName == ".PKGINFO") {
val buffer = new Array[Byte](archiveEntry.getSize.toInt)
is.read(buffer)
val metadata = Source.fromBytes(buffer)
.getLines()
.map(_.trim)
.filter(!_.isEmpty)
.filter(!_.startsWith("#"))
.map(line => {
val equals = line.indexOf("=")
if (equals < 0) {
throw new RuntimeException(s"Error parsing .PKGINFO file in '${file}'")
}
else {
(line.substring(0, equals).trim, line.substring(equals + 1, line.length).trim)
}
})
.toStream
.groupBy(_._1)
.map(pair => pair._1 -> pair._2.map(_._2).toList)
val data = new PkgData
for (pair <- metadata) {
val (key, value) = pair
key match {
case "size" =>
data.size = value.head.toLong
case "arch" =>
data.arch = value.head
case "replaces" =>
data.replaces = value.toSet.asJava
case "packager" =>
data.packager = value.head
case "url" =>
data.url = value.head
case "pkgname" =>
data.name = {
val name = new PkgName
name.id = value.head
name
}
case "builddate" =>
data.buildDate = new Date(value.head.toLong * 1000)
case "license" =>
data.license = value.head
case "pkgver" =>
data.version = value.head
case "pkgdesc" =>
data.description = value.head
case "provides" =>
data.provides = value.toSet.asJava
case "conflict" =>
data.conflict = value.toSet.asJava
case "backup" =>
data.backup = value.toSet.asJava
case "optdepend" =>
data.optdepend = value.toSet.asJava
case "depend" =>
data.depend = value.toSet.asJava
case "makedepend" =>
data.makedepend = value.toSet.asJava
case "makepkgopt" =>
data.makeopkgopt = value.toSet.asJava
case "pkgbase" =>
data.base = value.head
case _ =>
}
}
data.md5sum = hasher.getHashString(new FileInputStream(file))
data.fileName = file.getName
return data
}
else {
archiveEntry = is.getNextEntry()
}
}
throw new RuntimeException(s".PKGINFO file not found in '${file}'")
} finally {
is.close()
}
}
}

80
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgData.scala Normal file
View File

@@ -0,0 +1,80 @@
package net.woggioni.jpacrepo.pacbase
import java.util
import java.util.Date
import javax.persistence._
import javax.xml.bind.annotation.XmlRootElement
@Entity
@Access(AccessType.FIELD)
@XmlRootElement
@NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id")
@Table(indexes = Array(
new Index(columnList = "md5sum", unique = true),
new Index(columnList = "fileName", unique = true))
)
class PkgData {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
var id: Integer = _
@ManyToOne(cascade = Array(CascadeType.PERSIST), fetch = FetchType.EAGER)
var name: PkgName = _
var base: String = _
var version: String = _
var description: String = _
var url: String = _
@Temporal(TemporalType.TIMESTAMP)
var buildDate: Date = _
var packager: String = _
var size = 0L
var arch: String = _
var license: String = _
var md5sum: String = _
var fileName: String = _
@ElementCollection(fetch = FetchType.EAGER)
var replaces: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var conflict: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var provides: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var depend: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var optdepend: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var makedepend: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var makeopkgopt: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var backup: util.Set[String] = _
@Temporal(TemporalType.TIMESTAMP)
var updTimestamp: Date = _
@PreUpdate
@PrePersist
private def writeTimestamp(): Unit = updTimestamp = new Date
}

12
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgName.scala Normal file
View File

@@ -0,0 +1,12 @@
package net.woggioni.jpacrepo.pacbase
import javax.persistence.{Access, AccessType, Entity, Id}
import javax.xml.bind.annotation.XmlRootElement
@Entity
@Access(AccessType.FIELD)
@XmlRootElement
class PkgName {
@Id
var id : String = null
}

175
src/main/scala/net/woggioni/jpacrepo/service/PacmanServiceEJB.scala Normal file
View File

@@ -0,0 +1,175 @@
package net.woggioni.jpacrepo.service
import java.io.File
import java.nio.file.Files
import java.util
import java.util.{Calendar, Date}
import javax.ejb._
import javax.inject.Inject
import javax.persistence._
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import net.woggioni.jpacrepo.persistence.QueryEngine
import org.slf4j.Logger
import scala.collection.JavaConverters._
@Remote
trait PacmanServiceRemote {
def syncDB(): Unit
def deletePackage(filename: String): Unit
}
@Local
trait PacmanServiceView extends PacmanServiceRemote {
def countResults(name: String, version: String, arch: String): Long
def searchPackage(name: String, version: String, arch: String, page: Int, pageSize: Int, fileName : String): util.List[PkgData]
}
@Startup
@Singleton
@Lock(LockType.READ)
@TransactionManagement(TransactionManagementType.CONTAINER)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local(Array(classOf[PacmanServiceView]))
@Remote(Array(classOf[PacmanServiceRemote]))
class PacmanServiceEJB extends PacmanServiceView {
@Inject
private var emf: EntityManagerFactory = _
@Inject
private var ctx: ApplicationContext = _
@Inject
private var logger: Logger = _
final private val nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name"
final private val hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"
final private val hashQueryCount = "SELECT count(pdata) FROM PkgData pdata WHERE md5sum = :md5sum"
private def deletePkgData(em: EntityManager, pkgData: PkgData): Unit = {
if (pkgData.depend != null) pkgData.depend.asScala.foreach(em.remove)
if (pkgData.replaces != null) pkgData.replaces.asScala.foreach(em.remove)
if (pkgData.conflict != null) pkgData.conflict.asScala.foreach(em.remove)
if (pkgData.provides != null) pkgData.provides.asScala.foreach(em.remove)
if (pkgData.makedepend != null) pkgData.makedepend.asScala.foreach(em.remove)
if (pkgData.makeopkgopt != null) pkgData.makeopkgopt.asScala.foreach(em.remove)
if (pkgData.backup != null) pkgData.backup.asScala.foreach(em.remove)
em.remove(pkgData)
}
@Asynchronous
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Schedule(hour = "10", minute = "34", persistent = false)
override def syncDB() = {
val em = emf.createEntityManager
logger.info("Starting repository cleanup")
//Elimina i pacchetti sul DB che non esistono più nel filesystem
logger.info("Searching for packages that are no more in the filesystem")
val listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String]).getResultList
logger.info("Got list of filenames from db")
val knownPkg = listaDB
.asScala
.filter(fileName => {
val file = ctx.getFile(fileName)
val result = file.exists()
if (!result) {
logger.info(s"Removing package ${file.getName} which was not found in filesystem")
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
.setParameter("fileName", file.getName)
.getResultList.asScala.foreach((pkgData: PkgData) => deletePkgData(em, pkgData))
}
result
}).toSet
logger.info("Searching for new packages or packages that were modified after being added to the database")
new File(ctx.repoFolder)
.listFiles(_.getName.endsWith(".pkg.tar.xz"))
.foreach(file => {
if (!knownPkg.contains(file.getName) || {
val query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", classOf[Date])
query.setParameter("filename", file.getName)
val result = query.getSingleResult
file.lastModified > result.getTime}) {
try {
parseFile(em, file)
} catch {
case e: Exception =>
logger.error(s"Error parsing '${file.getAbsolutePath}'", e)
}
}
}
)
logger.info("Removing obsolete packages")
deleteOld(em)
logger.info("Repository cleanup completed successfully")
ctx.invalidateCache = true
}
private def parseFile(em: EntityManager, file: File) = {
val hquery = em.createQuery(hashQueryCount, classOf[java.lang.Long])
val data = Parser.parseFile(file)
hquery.setParameter("md5sum", data.md5sum)
if(hquery.getSingleResult == 0) {
val fquery = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
fquery.setParameter("fileName", file.getName)
fquery.getResultList.forEach((pkgData: PkgData) => deletePkgData(em, pkgData))
data.name = em.find(classOf[PkgName], data.name.id) match {
case null => data.name
case name : PkgName => name
}
em.persist(data)
logger.info(s"Persisting package ${file.getName}")
}
}
@TransactionAttribute(TransactionAttributeType.REQUIRED)
override def deletePackage(filename: String): Unit = {
val em = emf.createEntityManager
deletePackage(em, filename)
logger.info(s"Package $filename has been deleted")
}
private def deletePackage(em: EntityManager, filename: String): Unit = {
val fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", classOf[PkgData])
fquery.setParameter("fileName", filename)
val savedFiles = fquery.getResultList
if (savedFiles.size == 0) {
throw new RuntimeException(String.format("Package with name %s not found", filename))
}
val pkg = fquery.getResultList.get(0)
Files.delete(ctx.getFile(pkg).toPath)
em.remove(pkg)
}
final private val deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" + "(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)"
private def deleteOld(em: EntityManager): Unit = {
val query = em.createQuery(deleteQuery, classOf[String])
val cutoff = Calendar.getInstance
cutoff.add(Calendar.YEAR, -2)
query.setParameter("cutoff", cutoff.getTime)
query.setParameter("minVersions", 2.toLong)
val list = query.getResultList
list.forEach(deletePackage(_))
}
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
override def countResults(name: String, version: String, arch: String): Long = {
val em = emf.createEntityManager
QueryEngine.countResults(em, name, version, arch)
}
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
override def searchPackage(name: String, version: String, arch: String, pageNumber: Int, pageSize: Int, fileName : String) = {
val em = emf.createEntityManager
QueryEngine.searchPackage(em, name, version, arch, pageNumber, pageSize, null)
}
}

410
src/main/scala/net/woggioni/jpacrepo/service/PacmanWebService.scala Normal file
View File

@@ -0,0 +1,410 @@
package net.woggioni.jpacrepo.service
import java.io._
import java.net.URI
import java.nio.file.Files
import java.nio.file.StandardCopyOption.ATOMIC_MOVE
import java.util
import javax.ejb._
import javax.inject.Inject
import javax.persistence._
import javax.ws.rs._
import javax.ws.rs.core._
import javax.xml.bind.annotation.{XmlElement, XmlRootElement, XmlSeeAlso}
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{PkgData, PkgName}
import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
import org.slf4j.Logger
import scala.beans.BeanProperty
import scala.collection.JavaConverters._
import scala.collection.SortedMap
import scala.collection.immutable.TreeMap
object PacmanWebService {
private val nameQuery: String = "SELECT pname FROM PkgName pname WHERE id = :name"
private val fileNameQuery: String = "SELECT pdata FROM PkgData pdata WHERE fileName = :fileName"
private val hashQuery: String = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"
}
@XmlRootElement
@XmlSeeAlso(Array(classOf[PkgData]))
class PkgList() extends util.ArrayList[PkgData] {
def this(c: PkgData*) {
this()
c.foreach(el => add(el))
}
@XmlElement(name = "PkgData")
def getPackages: util.List[PkgData] = this
def setPackages(pkgs: util.List[PkgData]): Unit = {
this.clear()
this.addAll(pkgs)
}
}
@XmlRootElement
class StringList() extends util.ArrayList[String] {
def this(c: String*) {
this()
c.foreach(el => add(el))
}
@XmlElement(name = "string")
def getPackages: util.List[String] = this
def setPackages(pkgs: util.List[String]): Unit = {
this.clear()
this.addAll(pkgs)
}
}
@XmlRootElement
class PkgTuple {
@BeanProperty
var md5sum: String = _
@BeanProperty
var filename: String = _
@BeanProperty
var size: Long = _
}
@Singleton
@Path("/pkg")
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@Consumes(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@TransactionManagement(TransactionManagementType.CONTAINER)
class PacmanWebService {
private var cachedMap: SortedMap[(String, String, String), PkgTuple] = _
@Inject
private var emf: EntityManagerFactory = _
@Context
private var uriInfo: UriInfo = _
@Inject
private var log: Logger = _
@Inject
private var service : PacmanServiceView = _
@Inject
private var ctx: ApplicationContext = _
private def getCachedMap: SortedMap[(String, String, String), PkgTuple] = {
if (ctx.invalidateCache) {
val em = emf.createEntityManager()
val query = em.createQuery(
"SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.name.id, pkg.version, pkg.arch",
classOf[Array[AnyRef]])
val stream = query.getResultList
.asScala
.toStream
.map(pkg => {
val name: String = pkg(0).asInstanceOf[String]
val version: String = pkg(1).asInstanceOf[String]
val arch: String = pkg(2).asInstanceOf[String]
val filename: String = pkg(3).asInstanceOf[String]
val size: Long = pkg(4).asInstanceOf[Long]
val md5sum: String = pkg(5).asInstanceOf[String]
val tuple: PkgTuple = new PkgTuple
tuple.filename = filename
tuple.size = size
tuple.md5sum = md5sum
(name, version, arch) -> tuple
})
cachedMap = TreeMap(stream: _*)
ctx.invalidateCache = false
}
cachedMap
}
@GET
@Path("searchByName/{name}")
def searchByName(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST)
val query: String = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name)
Response.ok(em.createQuery(query, classOf[String]).getResultList).build
}
@GET
@Path("searchByHash/{md5sum}")
def searchByHash(@PathParam("md5sum") md5sum: String): Response = getPackageByHash(md5sum)
@GET
@Path("list/{name}")
def getPackage(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[_] = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", classOf[String])
query.setParameter("name", name)
Response.ok(query.getResultList).build
}
@GET
@Path("list/{name}/{version}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[_] = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", classOf[String])
query.setParameter("name", name)
query.setParameter("version", version)
Response.ok(query.getResultList).build
}
@GET
@Path("list/{name}/{version}/{arch}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String, @PathParam("arch") arch: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[PkgData] = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.name.id = :name AND " + "pkg.version = :version AND " + "pkg.arch = :arch " + "ORDER BY pkg.arch", classOf[PkgData])
query.setParameter("name", name)
query.setParameter("version", version)
query.setParameter("arch", arch)
Response.ok(query.getSingleResult).build
}
@GET
@Produces(Array(MediaType.APPLICATION_JSON))
@Path("map")
def getPackageMap(@Context request: Request): Response = {
val cc: CacheControl = new CacheControl
cc.setMaxAge(86400)
cc.setMustRevalidate(true)
cc.setNoCache(true)
val result : util.Map[String, util.Map[String, util.Map[String, PkgTuple]]] = getCachedMap.toStream
.groupBy(_._1._1).mapValues(
_.groupBy(_._1._2).mapValues(
_.map(pair => pair._1._3 -> pair._2).toMap.asJava
).asJava
).asJava
val etag: EntityTag = new EntityTag(Integer.toString(result.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
builder = Response.ok(result)
builder.tag(etag)
}
builder.cacheControl(cc)
builder.build
}
@GET
@Path("hashes")
def getHashes: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.md5sum FROM PkgData p", classOf[String])
val hl = new StringList(query.getResultList.asScala :_*)
Response.ok(hl).build
}
@GET
@Path("files")
def getFiles: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String])
val hl = new StringList(query.getResultList.asScala :_*)
Response.ok(hl).build
}
private def getPackageByHash(md5sum: String): Response = {
val em = emf.createEntityManager()
val hquery = em.createQuery(PacmanWebService.hashQuery, classOf[PkgData])
if (md5sum != null) hquery.setParameter("md5sum", md5sum)
manageQueryResult(hquery.getResultList, true)
}
private def getPackageByFileName(file: String): Response = {
val em = emf.createEntityManager()
val fnquery: TypedQuery[PkgData] = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
fnquery.setParameter("fileName", file)
manageQueryResult(fnquery.getResultList, true)
}
@GET
@Path("filesize/{filename}")
def getFileSize(@PathParam("filename") fileName: String, @Context request: Request): Response = {
val cc: CacheControl = new CacheControl
cc.setMaxAge(86400)
cc.setMustRevalidate(true)
cc.setNoCache(true)
val etag: EntityTag = new EntityTag(Integer.toString(getCachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val res: File = ctx.getFile(fileName)
if (!res.exists) throw new NotFoundException(String.format("File '%s' was not found", fileName))
builder = Response.ok(res.length)
builder.tag(etag)
}
builder.cacheControl(cc)
builder.build
}
@GET
@Path("download/{filename}")
@Produces(Array(MediaType.APPLICATION_OCTET_STREAM)) def downloadPackage(@PathParam("filename") fileName: String): Response = {
val em = emf.createEntityManager()
val fnquery = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
fnquery.setParameter("fileName", fileName)
try {
val pkg: PkgData = fnquery.getSingleResult
val stream: StreamingOutput = (output: OutputStream) => {
val input: FileInputStream = new FileInputStream(ctx.getFile(pkg))
try {
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ({
read = input.read(bytes)
read >= 0
}) {
output.write(bytes, 0, read)
}
} finally {
input.close()
}
}
return Response.ok(stream).header("Content-Length", ctx.getFile(pkg).length).build
} catch {
case _: NoResultException =>
throw new NotFoundException
}
}
@POST
@Path("/doYouWantAny")
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
def doYouWantAny(filenames: util.List[String]): Response = {
val em = emf.createEntityManager()
val result = Set(filenames.asScala: _*)
if (result.nonEmpty) {
val query = em.createQuery("SELECT pkg.fileName from PkgData pkg WHERE pkg.fileName in :filenames", classOf[String])
query.setParameter("filenames", filenames)
val toBeRemoved = Set(query.getResultList.asScala: _*)
Response.ok((result -- toBeRemoved).toArray).build
}
else {
Response.ok(result.toArray).build
}
}
@POST
@Path("/upload")
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Consumes(Array(MediaType.APPLICATION_OCTET_STREAM))
def createPackage(input: InputStream, @MatrixParam("filename") filename: String): Response = {
val em = emf.createEntityManager()
if (filename == null) throw new BadRequestException
val file: File = File.createTempFile(filename, "tmp", new File(ctx.repoFolder))
val fquery: TypedQuery[PkgData] = em.createQuery(PacmanWebService.fileNameQuery, classOf[PkgData])
fquery.setParameter("fileName", filename)
val savedFiles: util.List[PkgData] = fquery.getResultList
if (savedFiles.size > 0) Response.notModified.build
else {
val fos: FileOutputStream = new FileOutputStream(file)
try {
val buffer: Array[Byte] = new Array[Byte](4096)
var read = 0
while ({
read = input.read(buffer)
read >= 0}) {
fos.write(buffer, 0, read)
}
val pkg = Parser.parseFile(file)
pkg.fileName = filename
val nquery: TypedQuery[PkgName] = em.createQuery(PacmanWebService.nameQuery, classOf[PkgName])
nquery.setParameter("name", pkg.name.id)
val savedName: util.List[PkgName] = nquery.getResultList
if (savedName.size > 0) pkg.name = savedName.get(0)
em.persist(pkg)
log.info(s"Persisting package ${pkg.fileName}")
val pkgUri: URI = uriInfo.getAbsolutePathBuilder.path(pkg.fileName).build()
Files.move(file.toPath, new File(ctx.repoFolder, filename).toPath, ATOMIC_MOVE)
ctx.invalidateCache = true
cachedMap = null
Response.created(pkgUri).build
} catch {
case e: Exception =>
Files.delete(file.toPath)
throw e
} finally {
fos.close()
}
}
}
@GET
@Path("/search")
def searchPackage(
@QueryParam("name") name: String,
@QueryParam("version") version: String,
@QueryParam("arch") arch: String,
@QueryParam("page") pageNumber: Int,
@QueryParam("pageSize") pageSize: Int,
@QueryParam("fileName") fileName: String): util.List[PkgData] = {
service.searchPackage(name, version,arch, pageNumber, pageSize, fileName)
}
@OPTIONS
@Path("/downloadTar")
@Produces(Array("text/plain; charset=UTF-8"))
def options: Response = Response.ok("POST, OPTIONS").build
@POST
@Path("/downloadTar")
@Produces(Array("application/x-tar"))
@Consumes(Array(MediaType.APPLICATION_FORM_URLENCODED))
def downloadTar(@FormParam("pkgs") formData: String): Response = {
val files: Array[String] = formData.split(" ")
files.find(!ctx.getFile(_).exists) match {
case Some(fileName) => throw new NotFoundException(s"Package file '$fileName' does not exist")
case None =>
}
val stream = new StreamingOutput() {
override def write(output: OutputStream): Unit = {
val taos: TarArchiveOutputStream = new TarArchiveOutputStream(output)
try {
for (fname <- files) {
val file: File = ctx.getFile(fname)
val input: FileInputStream = new FileInputStream(file)
val entry: TarArchiveEntry = new TarArchiveEntry(fname)
entry.setSize(file.length)
taos.putArchiveEntry(entry)
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ({
read = input.read(bytes)
read >= 0}) {
taos.write(bytes, 0, read)
}
taos.closeArchiveEntry()
}
} finally {
taos.close()
}
}
}
return Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build
}
private def manageQueryResult(list: util.List[PkgData]): Response = {
return manageQueryResult(list, false)
}
private def manageQueryResult(list: util.List[PkgData], singleResult: Boolean): Response = {
val pkgList: PkgList = new PkgList(list.asScala :_*)
if (pkgList.size == 0) throw new NotFoundException
else if (singleResult) if (pkgList.size == 1) Response.ok(pkgList.get(0)).build
else throw new NonUniqueResultException("The returned list does not contain a single element")
else Response.ok(pkgList).build
}
}

34
src/test/java/ClientTest.java
View File

@@ -1,13 +1,12 @@
import com.oggio88.jpacrepo.service.PacmanServiceRemote;
import com.thoughtworks.xstream.XStream;
import net.woggioni.jpacrepo.model.Hasher;
import net.woggioni.jpacrepo.model.MD5InputStream;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.PkgData;
import net.woggioni.jpacrepo.service.PacmanServiceRemote;
import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
import org.jboss.resteasy.spi.ResteasyProviderFactory;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.pacbase.Hasher;
import com.oggio88.jpacrepo.pacbase.MD5InputStream;
import com.oggio88.jpacrepo.pacbase.Parser;
import com.oggio88.jpacrepo.service.PacmanServiceView;
import org.junit.Test;
import javax.naming.*;
@@ -22,16 +21,12 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.List;
import java.util.Properties;
/**
* Created by walter on 29/03/15.
*/
public class ClientTest
{
@Test
public void testGET() throws Exception
public void testGET()
{
ResteasyProviderFactory instance = ResteasyProviderFactory.getInstance();
RegisterBuiltin.register(instance);
@@ -92,14 +87,14 @@ public class ClientTest
System.out.println(Hasher.computeMD5(new FileInputStream(file)));
InputStream fis = new FileInputStream(file);
MD5InputStream h = Hasher.createMD5InputStream(fis);
MD5InputStream h = new MD5InputStream(fis);
long a = new File(file).length();
byte[] out = new byte[(int)a];
h.read(out, 0, (int) a);
System.out.println(h.digest());
PkgData p = new Parser().parseFile(new File(file));
System.out.println(p.md5sum);
PkgData p = Parser.parseFile(new File(file));
System.out.println(p.md5sum());
}
@@ -125,26 +120,27 @@ public class ClientTest
@Test
public void invokeStatelessBean() throws Exception
{
// Let's lookup the remote stateless calculator
Properties prop = new Properties();
InputStream in = getClass().getClassLoader().getResourceAsStream("jboss-ejb-client.properties");
prop.load(in);
prop.put(Context.URL_PKG_PREFIXES, "org.jboss.ejb.client.naming");
prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory");
// prop.put(Context.PROVIDER_URL, "http-remoting://localhost:8080");
prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
prop.put(Context.PROVIDER_URL, "http-remoting://localhost:8080");
// prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
// prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
prop.put(Context.SECURITY_PRINCIPAL, "walter");
prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d");
// prop.put(Context.SECURITY_PRINCIPAL, "admin");
// prop.put(Context.SECURITY_CREDENTIALS, "123456");
prop.put("jboss.naming.client.ejb.context", true);
Context context = new InitialContext(prop);
Context ctx = new InitialContext(prop);
traverseJndiNode("/", context);
// final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
final PacmanServiceRemote service = (PacmanServiceRemote) ctx.lookup(
"/jpacrepo-2.0/PacmanServiceEJB!com.oggio88.jpacrepo.service.PacmanServiceRemote"
"/jpacrepo_2.12-2.0/PacmanServiceEJB!net.woggioni.jpacrepo.service.PacmanServiceRemote"
);
// List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
// System.out.println(new XStream().toXML(pkgs));

13
src/test/java/ParseTest.java
View File

@@ -1,13 +1,12 @@
import com.thoughtworks.xstream.XStream;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.PkgData;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.pacbase.Parser;
import org.junit.Test;
import java.io.File;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -18,14 +17,14 @@ import java.util.List;
public class ParseTest
{
// @Test
public void test() throws Exception
public void test()
{
Collection<File> ls = FileUtils.listFiles(new File("/var/cache/pacman/pkg"), new RegexFileFilter(".*\\.pkg\\.tar\\.xz"), DirectoryFileFilter.DIRECTORY);
int i = 0;
List<PkgData> lista = new ArrayList<>();
for (File file : ls)
{
PkgData data = new Parser().parseFile(file);
PkgData data = Parser.parseFile(file);
lista.add(data);
//System.out.println(new XStream().toXML(data));
// if(i++>10) break;
@@ -34,13 +33,13 @@ public class ParseTest
}
@Test
public void parseTest() throws Exception
public void parseTest()
{
String[] files = new String[]{"/home/walter/Scaricati/oh-my-zsh-git-3912.d310fac-1-any.pkg.tar.xz"};
for (String file : files)
{
PkgData data = new Parser().parseFile(new File(file));
PkgData data = Parser.parseFile(new File(file));
System.out.println(new XStream().toXML(data));
}
}

16
src/test/java/net/woggioni/jpacrepo/annotation/UnitTesting.java Normal file
View File

@@ -0,0 +1,16 @@
package net.woggioni.jpacrepo.annotation;
import javax.enterprise.inject.Alternative;
import javax.enterprise.inject.Stereotype;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import static java.lang.annotation.ElementType.TYPE;
import static java.lang.annotation.RetentionPolicy.RUNTIME;
@Alternative
@Stereotype
@Retention(RUNTIME)
@Target(TYPE)
public @interface UnitTesting {
}

6
src/test/resources/META-INF/beans.xml Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://xmlns.jcp.org/xml/ns/javaee">
<alternatives>
<stereotype>net.woggioni.jpacrepo.annotation.UnitTesting</stereotype>
</alternatives>
</beans>

24
src/test/resources/META-INF/persistence.xml Normal file
View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<persistence xmlns="http://xmlns.jcp.org/xml/ns/persistence"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/persistence http://xmlns.jcp.org/xml/ns/persistence/persistence_2_1.xsd"
version="2.1">
<persistence-unit name="test" transaction-type="RESOURCE_LOCAL">
<provider>org.hibernate.jpa.HibernatePersistenceProvider</provider>
<class>net.woggioni.jpacrepo.pacbase.PkgData</class>
<class>net.woggioni.jpacrepo.pacbase.PkgName</class>
<properties>
<property name="javax.persistence.jdbc.driver" value="org.h2.Driver"/>
<property name="javax.persistence.jdbc.url" value="jdbc:h2:mem:jpwdb;create=true;FILE_LOCK=NO"/>
<property name="javax.persistence.jdbc.user" value="APP"/>
<property name="javax.persistence.jdbc.password" value="APP"/>
<property name="org.jboss.logging.provider" value="log4j2"/>
<property name="javax.persistence.schema-generation.database.action" value="create"/>
<property name="hibernate.dialect" value="org.hibernate.dialect.HSQLDialect"/>
<property name="hibernate.connection.isolation" value="1"/>
</properties>
</persistence-unit>
</persistence>

17
src/test/resources/log4j2.xml Normal file
View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
<Properties>
<Property name="log-path">$${env:APOLLO_ENVIRONMENT_ROOT:-build}/var/logs</Property>
</Properties>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss,SSS} %highlight{[%p]} (%t) %c: %m%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="INFO">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>

11
src/test/scala/net/woggioni/jpacrepo/persistence/TestPersistenceProducer.scala Normal file
View File

@@ -0,0 +1,11 @@
package net.woggioni.jpacrepo.persistence
import javax.enterprise.inject.Produces
import javax.persistence.Persistence
import net.woggioni.jpacrepo.annotation.UnitTesting
class TestPersistenceProducer {
@Produces
@UnitTesting
def createEntityManagerFactory = Persistence.createEntityManagerFactory("test")
}

61
src/test/scala/net/woggioni/jpacrepo/service/PacmanServiceEJBTest.scala Normal file
View File

@@ -0,0 +1,61 @@
package net.woggioni.jpacrepo.service
import javax.enterprise.util.TypeLiteral
import net.woggioni.jpacrepo.context.ApplicationContext
import net.woggioni.jpacrepo.factory.BeanFactory
import net.woggioni.jpacrepo.persistence.TestPersistenceProducer
import org.jboss.weld.environment.se.Weld
import org.junit.Test
//object WeldContainer {
// private val weld = new Weld
// weld.disableDiscovery()
//
// weld.beanClasses(
// classOf[TestPersistenceProducer],
// classOf[PacmanServiceEJB],
// classOf[ApplicationContext],
// classOf[BeanFactory],
// )
//
// private val count = new AtomicInteger(0)
//}
//class WeldContainer extends AutoCloseable {
//
// val container = WeldContainer.weld.initialize()
// WeldContainer.count.incrementAndGet()
//
// override def close(): Unit = {
// container.close()
// if(WeldContainer.count.decrementAndGet() == 0) {
//// WeldContainer.weld.shutdown()
// }
// }
//}
class PacmanServiceEJBTest {
private val weld = new Weld
// weld.disableDiscovery()
weld.alternatives(classOf[TestPersistenceProducer])
////
weld.beanClasses(
classOf[PacmanServiceEJB],
classOf[PacmanWebService],
// classOf[ApplicationContext],
classOf[BeanFactory],
)
@Test
def test = {
val container = weld.initialize()
try {
val s = getClass.getResourceAsStream("/log4j2.xml")
val service = container.select(new TypeLiteral[PacmanServiceView] {}).get()
service.syncDB()
} finally {
container.close()
}
}
}

29
src/test/scala/net/woggioni/jpacrepo/service/PacmanWebServiceTest.scala Normal file
View File

@@ -0,0 +1,29 @@
package net.woggioni.jpacrepo.service
class PacmanWebServiceTest {
// val server = {
// val server = new UndertowJaxrsServer()
// server.start()
// import org.jboss.resteasy.spi.ResteasyDeployment
// val deployment = new ResteasyDeployment
// deployment.setInjectorFactoryClass("org.jboss.resteasy.cdi.CdiInjectorFactory")
// deployment.setApplicationClass(classOf[ApplicationConfig].getName)
// val di: DeploymentInfo = server.undertowDeployment(deployment)
// di.setClassLoader(classOf[ApplicationConfig].getClassLoader)
// di.setContextPath("/jpacrepo")
// di.setDeploymentName("jpacrepo")
// di.addListeners(Servlets.listener(classOf[Listener]))
// server.deploy(di)
// server
// }
// @Test
// def foo {
// val client = ClientBuilder.newClient()
// val webTarget = client.target(TestPortProvider.generateURL("/jpacrepo/rest/pkg/map"))
// val response = webTarget.request().get()
// val res = response.getEntity
// response.getStatus
// }
}