diff --git a/build.sbt b/build.sbt index 596cf46..62fdd49 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,7 @@ scalaVersion := "2.12.2" libraryDependencies += "org.tukaani" % "xz" % "1.6" libraryDependencies += "org.apache.commons" % "commons-compress" % "1.14" -libraryDependencies += "org.projectlombok" % "lombok" % "1.16.18" % "provided" +libraryDependencies += "org.projectlombok" % "lombok" % "1.16.18" libraryDependencies += "javax" % "javaee-api" % "7.0" % "provided" //libraryDependencies += "javax" % "javaee-api" % "7.0 " @@ -42,6 +42,8 @@ artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) => artifact.name + "-" + module.revision + "." + artifact.extension } +//javacOptions += "-g" + val wildflyUsername = SettingKey[String]("wildfly-username", "The account username to use to connect to wildfly with jboss-cli.sh") val wildflyPassword = SettingKey[String]("wildfly-password", "The account password to use to connect to wildfly with jboss-cli.sh") val wildflyURL = SettingKey[String]("wildfly-url", "The username to use to connect to wildfly with jboss-cli.sh") diff --git a/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java b/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java index c2fd68d..44f24ae 100644 --- a/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java +++ b/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java @@ -28,7 +28,6 @@ public class ApplicationContext private String repoFolder; - @Getter @Setter private PacmanServiceView pacmanService; public ApplicationContext(String propertyFile) @@ -44,17 +43,20 @@ public class ApplicationContext // get the property value and print it out systemProperties.getProperty("RepoFolder"); repoFolder = systemProperties.getProperty("RepoFolder"); - } catch (IOException ex) + } + catch (IOException ex) { throw new RuntimeException(ex); - } finally + } + finally { if (input != null) { try { input.close(); - } catch (IOException e) + } + catch (IOException e) { throw new RuntimeException(e); } @@ -71,4 +73,20 @@ public class ApplicationContext { return new File(new File(repoFolder), pkg.fileName); } + + public File getFile(String fileName) + { + return new File(new File(repoFolder), fileName); + } + + public PacmanServiceView getPacmanService() + { + return pacmanService; + } + + public void setPacmanService(PacmanServiceView pacmanService) + { + this.pacmanService = pacmanService; + } + } diff --git a/src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java b/src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java index a80a17c..2290923 100644 --- a/src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java +++ b/src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java @@ -28,7 +28,7 @@ public class QueryEngine public QueryEngine select(String... fields) { String[] strarr = new String[fields.length]; - for(int i=0; i knownPkg; + private Set knownPkg; + @Asynchronous @Schedule(hour = "3", minute = "00", persistent = false) public void syncDB() { logger.log(Level.INFO, "Starting repository cleanup"); - knownPkg = new HashMap<>(); + knownPkg = new HashSet<>(); //Elimina i pacchetti sul DB che non esistono più nel filesystem + logger.info("Searching for packages that are no more in the filesystem"); try { ut.setTransactionTimeout(1000); ut.begin(); - List listaDB = em.createQuery("SELECT p FROM PkgData p", PkgData.class).getResultList(); - for (PkgData p : listaDB) + List listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList(); + logger.info("Got list of filenames from db"); + for (String fileName : listaDB) { - knownPkg.put(p.fileName, p); - File file = ctx.getFile(p); + File file = ctx.getFile(fileName); if (!file.exists()) { logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName())); - em.remove(p); + PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", + PkgData.class).setParameter("fileName", file.getName()).getSingleResult(); + em.remove(pkg); + } + else + { + knownPkg.add(fileName); } } ut.commit(); - } catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e) + } + catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e) { throw new RuntimeException(e); } - //Aggiunge sul DB i pacchetti presenti nel filesystem - int cores = Runtime.getRuntime().availableProcessors(); + logger.info("Searching for new packages or packages that were modified after being added to the database"); + List pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz")))); - List ls = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz")))); - Deque stack = new LinkedList<>(ls); - Thread[] works = new Thread[cores]; - for(int i=0; i query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class); + query.setParameter("filename", file.getName()); + Date result = query.getSingleResult(); + if(file.lastModified() > result.getTime()) + { + update = true; + } } - catch (InterruptedException e) + + if (update) { - e.printStackTrace(); + try + { + ut.begin(); + parseFile(parser, file); + ut.commit(); + } + catch (Exception e) + { + logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath())); + try + { + ut.rollback(); + } + catch (SystemException e1) + { + throw new RuntimeException(e1); + } + throw new RuntimeException(e); + } } } + logger.log(Level.INFO, "Removing obsolete packages"); deleteOld(); + logger.info("Repository cleanup completed successfully"); } private void parseFile(Parser p, File file) throws Exception @@ -114,7 +143,7 @@ public class PacmanServiceEJB implements PacmanServiceView PkgData data = p.parseFile(file); hquery.setParameter("md5sum", data.md5sum); - List savedFiles = hquery.getResultList(); + List savedFiles = hquery.getResultList(); if (savedFiles.size() > 0) { return; @@ -124,7 +153,7 @@ public class PacmanServiceEJB implements PacmanServiceView TypedQuery fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class); fquery.setParameter("fileName", file.getName()); savedFiles = fquery.getResultList(); - if(savedFiles.size()>0) + if (savedFiles.size() > 0) { em.remove(savedFiles.get(0)); } @@ -148,7 +177,7 @@ public class PacmanServiceEJB implements PacmanServiceView TypedQuery fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class); fquery.setParameter("fileName", filename); List savedFiles = fquery.getResultList(); - if(savedFiles.size()==0) + if (savedFiles.size() == 0) { ut.rollback(); throw new RuntimeException(String.format("Package with name %s not found", filename)); @@ -166,58 +195,9 @@ public class PacmanServiceEJB implements PacmanServiceView em.remove(pkg); ut.commit(); - logger.log(Level.INFO,String.format("Package %s has been deleted", filename)); + logger.log(Level.INFO, String.format("Package %s has been deleted", filename)); } - private class SyncWorker implements Runnable - { - Deque ls; - - Parser parser = new Parser(); - - public SyncWorker(Deque ls) - { - this.ls = ls; - } - - @Override - public void run() - { - while(ls.size() > 0) - { - File file; - synchronized (ls) - { - file = ls.pop(); - } - try - { - PkgData p = knownPkg.get(file.getName()); - if (p == null || file.lastModified() > p.updTimestamp.getTime()) - { - ut.begin(); - parseFile(parser, file); - ut.commit(); - } - } - catch (Exception e) - { - logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath())); - try - { - ut.rollback(); - } - catch (SystemException e1) - { - throw new RuntimeException(e1); - } - throw new RuntimeException(e); - } - } - } - } - - private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" + "(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)"; @@ -227,10 +207,10 @@ public class PacmanServiceEJB implements PacmanServiceView Calendar cutoff = Calendar.getInstance(); cutoff.add(Calendar.YEAR, -2); query.setParameter("cutoff", cutoff.getTime()); - query.setParameter("minVersions", (long)2); + query.setParameter("minVersions", (long) 2); List list = query.getResultList(); - logger.log(Level.INFO, "Removing obsolete packages"); - list.forEach((el) -> { + list.forEach((el) -> + { try { deletePackage(el); @@ -253,25 +233,25 @@ public class PacmanServiceEJB implements PacmanServiceView builder = em.getCriteriaBuilder(); criteriaQuery = builder.createQuery(Long.class); entity = criteriaQuery.from(PkgData.class); - Predicate finalPredicate=null, p; + Predicate finalPredicate = null, p; - if(name != null && !name.isEmpty()) + if (name != null && !name.isEmpty()) { p = builder.equal(entity.get("name").get("id"), name); finalPredicate = p; } - if(version != null && !version.isEmpty()) + if (version != null && !version.isEmpty()) { - p=builder.equal(entity.get("version"), version); + p = builder.equal(entity.get("version"), version); finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; } - if(arch != null && !arch.isEmpty()) + if (arch != null && !arch.isEmpty()) { - p=builder.equal(entity.get("arch"), arch); + p = builder.equal(entity.get("arch"), arch); finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; } - if(finalPredicate != null) + if (finalPredicate != null) { criteriaQuery.select(builder.count(entity)).where(finalPredicate); } @@ -284,35 +264,35 @@ public class PacmanServiceEJB implements PacmanServiceView @Override @Lock(LockType.READ) - public List listProperty(String property, Map equalityConditions, Class cls) + public List listProperty(String property, Map equalityConditions, Class cls) { CriteriaBuilder builder = em.getCriteriaBuilder(); CriteriaQuery criteriaQuery = builder.createQuery(cls); Root entity = criteriaQuery.from(PkgData.class); - Predicate finalPredicate=null, p; + Predicate finalPredicate = null, p; String key = equalityConditions.get("name"); - if(key != null && !key.isEmpty()) + if (key != null && !key.isEmpty()) { p = builder.equal(entity.get("name").get("id"), key); finalPredicate = p; } key = equalityConditions.get("version"); - if(key != null && !key.isEmpty()) + if (key != null && !key.isEmpty()) { p = builder.equal(entity.get("version"), key); finalPredicate = p; } key = equalityConditions.get("arch"); - if(key != null && !key.isEmpty()) + if (key != null && !key.isEmpty()) { p = builder.equal(entity.get("arch"), key); finalPredicate = p; } - if(finalPredicate != null) + if (finalPredicate != null) { criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate); } @@ -335,25 +315,25 @@ public class PacmanServiceEJB implements PacmanServiceView builder = em.getCriteriaBuilder(); criteriaQuery = builder.createQuery(PkgData.class); entity = criteriaQuery.from(PkgData.class); - Predicate finalPredicate=null, p; + Predicate finalPredicate = null, p; - if(name != null && !name.isEmpty()) + if (name != null && !name.isEmpty()) { p = builder.equal(entity.get("name").get("id"), name); finalPredicate = p; } - if(version != null && !version.isEmpty()) + if (version != null && !version.isEmpty()) { - p=builder.equal(entity.get("version"), version); + p = builder.equal(entity.get("version"), version); finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; } - if(arch != null && !arch.isEmpty()) + if (arch != null && !arch.isEmpty()) { - p=builder.equal(entity.get("arch"), arch); + p = builder.equal(entity.get("arch"), arch); finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; } - if(finalPredicate != null) + if (finalPredicate != null) { criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName"))); } @@ -362,11 +342,11 @@ public class PacmanServiceEJB implements PacmanServiceView criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName"))); } TypedQuery query = em.createQuery(criteriaQuery); - if(pageNumber>=0) + if (pageNumber >= 0) { - query.setFirstResult(pageNumber*pageSize); + query.setFirstResult(pageNumber * pageSize); } - if(pageSize>0) + if (pageSize > 0) { query.setMaxResults(pageSize); } diff --git a/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java b/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java index 701b745..4f18700 100644 --- a/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java +++ b/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java @@ -9,5 +9,6 @@ import javax.ejb.Remote; @Remote public interface PacmanServiceRemote { + public void syncDB(); public void deletePackage(String filename) throws Exception; } diff --git a/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java b/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java index ff7056a..82491b9 100644 --- a/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java +++ b/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java @@ -1,11 +1,11 @@ package com.oggio88.jpacrepo.service; -import com.oggio88.jpacrepo.model.PkgName; import com.oggio88.jpacrepo.context.ApplicationContext; import com.oggio88.jpacrepo.context.DefaultConfiguration; -import com.oggio88.jpacrepo.model.StringList; import com.oggio88.jpacrepo.model.PkgData; import com.oggio88.jpacrepo.model.PkgList; +import com.oggio88.jpacrepo.model.PkgName; +import com.oggio88.jpacrepo.model.StringList; import com.oggio88.jpacrepo.pacbase.Parser; import com.oggio88.jpacrepo.persistence.QueryEngine; @@ -21,6 +21,8 @@ import java.io.*; import java.net.URI; import java.nio.file.Files; import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -71,7 +73,7 @@ public class PacmanWebService { return getPackageByFileName(filename); } - else if (name != null || arch != null || version != null) + else { QueryEngine qe = new QueryEngine(PkgData.class); @@ -82,10 +84,70 @@ public class PacmanWebService String query = qe.build(); return manageQueryResult(em.createQuery(query).getResultList()); } - else + } + + @GET + @Path("searchByName/{name}") + public Response searchPackagebyName(@PathParam("name") String name) + { + if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST); + String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name); + return Response.ok(em.createQuery(query, String.class).getResultList()).build(); + } + + @GET + @Path("list/{name}") + public Response getPackage(@PathParam("name") String name) + { + TypedQuery query = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", String.class); + query.setParameter("name", name); + return Response.ok(query.getResultList()).build(); + } + + @GET + @Path("list/{name}/{version}") + public Response getPackage(@PathParam("name") String name, @PathParam("version") String version) + { + TypedQuery query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", String.class); + query.setParameter("name", name); + query.setParameter("version", version); + return Response.ok(query.getResultList()).build(); + } + + @GET + @Path("list/{name}/{version}/{arch}") + public Response getPackage(@PathParam("name") String name, @PathParam("version") String version, @PathParam("arch") String arch) + { + TypedQuery query = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + + "pkg.name.id = :name AND " + + "pkg.version = :version AND " + + "pkg.arch = :arch " + + "ORDER BY pkg.arch", PkgData.class); + query.setParameter("name", name); + query.setParameter("version", version); + query.setParameter("arch", arch); + return Response.ok(query.getSingleResult()).build(); + } + + @GET + @Path("map") + public Response getPackageMap() + { + TypedQuery query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class); + SortedMap>> result = new TreeMap<>(); + for (Object[] pkg : query.getResultList()) { - throw new NotFoundException(); + String name = (String) pkg[0]; + String version = (String) pkg[1]; + String arch = (String) pkg[2]; + String filename = (String) pkg[3]; + result.putIfAbsent(name, new TreeMap<>()); + SortedMap> map = result.get(name); + map.putIfAbsent(version, new TreeMap<>()); + SortedMap map2 = map.get(version); + map2.putIfAbsent(arch, filename); } + return Response.ok(result).build(); } @GET @@ -240,7 +302,6 @@ public class PacmanWebService private Response manageQueryResult(List list, boolean singleResult) { -// log.log(Level.INFO, "size: " + list.size()); PkgList pkgList = new PkgList(list); if (pkgList.size() == 0) { diff --git a/src/test/java/ClientTest.java b/src/test/java/ClientTest.java index 6ff276d..6984890 100644 --- a/src/test/java/ClientTest.java +++ b/src/test/java/ClientTest.java @@ -1,3 +1,4 @@ +import com.oggio88.jpacrepo.service.PacmanServiceRemote; import com.thoughtworks.xstream.XStream; import org.jboss.resteasy.plugins.providers.RegisterBuiltin; import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider; @@ -133,18 +134,21 @@ public class ClientTest prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory"); // prop.put(Context.PROVIDER_URL, "http-remoting://localhost:8080"); - prop.put(Context.PROVIDER_URL, "http-remoting://odroid-xu4:8080"); + prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080"); // prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447"); - prop.put(Context.SECURITY_PRINCIPAL, "jpacrepo"); - prop.put(Context.SECURITY_CREDENTIALS, "password01."); + prop.put(Context.SECURITY_PRINCIPAL, "walter"); + prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d"); prop.put("jboss.naming.client.ejb.context", true); Context context = new InitialContext(prop); Context ctx = new InitialContext(prop); traverseJndiNode("/", context); // final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService"); - final PacmanServiceView service = (PacmanServiceView) ctx.lookup("/jpacrepo/PacmanServiceEJB!org.jpacrepo.service.PacmanService"); - List pkgs = service.searchPackage("google-earth", null, null, 1, 10); - System.out.println(new XStream().toXML(pkgs)); + final PacmanServiceRemote service = (PacmanServiceRemote) ctx.lookup( + "/jpacrepo-2.0/PacmanServiceEJB!com.oggio88.jpacrepo.service.PacmanServiceRemote" + ); +// List pkgs = service.searchPackage("google-earth", null, null, 1, 10); +// System.out.println(new XStream().toXML(pkgs)); + service.syncDB(); } }