added map webservice

fixed syncDB EJB method

build.sbt

@@ -9,7 +9,7 @@ scalaVersion := "2.12.2"
 
 libraryDependencies += "org.tukaani" % "xz" % "1.6"
 libraryDependencies += "org.apache.commons" % "commons-compress" % "1.14"
-libraryDependencies += "org.projectlombok" % "lombok" % "1.16.18" % "provided"
+libraryDependencies += "org.projectlombok" % "lombok" % "1.16.18"
 libraryDependencies += "javax" % "javaee-api" % "7.0" % "provided"
 //libraryDependencies += "javax" % "javaee-api" % "7.0 "
 
@@ -42,6 +42,8 @@ artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
     artifact.name + "-" + module.revision + "." + artifact.extension
 }
 
+//javacOptions += "-g"
+
 val wildflyUsername = SettingKey[String]("wildfly-username", "The account username to use to connect to wildfly with jboss-cli.sh")
 val wildflyPassword = SettingKey[String]("wildfly-password", "The account password to use to connect to wildfly with jboss-cli.sh")
 val wildflyURL = SettingKey[String]("wildfly-url", "The username to use to connect to wildfly with jboss-cli.sh")

src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java

@@ -28,7 +28,6 @@ public class ApplicationContext
 
     private String repoFolder;
 
-    @Getter @Setter
     private PacmanServiceView pacmanService;
 
     public ApplicationContext(String propertyFile)
@@ -44,17 +43,20 @@ public class ApplicationContext
             // get the property value and print it out
             systemProperties.getProperty("RepoFolder");
             repoFolder = systemProperties.getProperty("RepoFolder");
-        } catch (IOException ex)
+        }
+        catch (IOException ex)
         {
             throw new RuntimeException(ex);
-        } finally
+        }
+        finally
         {
             if (input != null)
             {
                 try
                 {
                     input.close();
-                } catch (IOException e)
+                }
+                catch (IOException e)
                 {
                     throw new RuntimeException(e);
                 }
@@ -71,4 +73,20 @@ public class ApplicationContext
     {
         return new File(new File(repoFolder), pkg.fileName);
     }
+
+    public File getFile(String fileName)
+    {
+        return new File(new File(repoFolder), fileName);
+    }
+
+    public PacmanServiceView getPacmanService()
+    {
+        return pacmanService;
+    }
+
+    public void setPacmanService(PacmanServiceView pacmanService)
+    {
+        this.pacmanService = pacmanService;
+    }
+
 }

src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java

@@ -28,7 +28,7 @@ public class QueryEngine
     public QueryEngine select(String... fields)
     {
         String[] strarr = new String[fields.length];
-        for(int i=0; i<fields.length; i++)
+        for (int i = 0; i < fields.length; i++)
         {
             strarr[i] = "e." + fields[i];
         }
@@ -50,6 +50,13 @@ public class QueryEngine
 
     public String build()
     {
-        return query + " WHERE " + String.join(" AND ", where);
+        if (where.isEmpty())
+        {
+            return query;
+        }
+        else
+        {
+            return query + " WHERE " + String.join(" AND ", where);
+        }
     }
 }

src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java

@@ -6,12 +6,14 @@ import com.oggio88.jpacrepo.model.PkgData;
 import com.oggio88.jpacrepo.model.PkgName;
 import com.oggio88.jpacrepo.pacbase.Parser;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 import javax.ejb.*;
 import javax.enterprise.concurrent.ManagedThreadFactory;
 import javax.inject.Inject;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
+import javax.persistence.Query;
 import javax.persistence.TypedQuery;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
@@ -29,6 +31,8 @@ import java.util.logging.Logger;
 @Singleton
 @TransactionManagement(TransactionManagementType.BEAN)
 @ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
+@Local(PacmanServiceView.class)
+@Remote(PacmanServiceRemote.class)
 public class PacmanServiceEJB implements PacmanServiceView
 {
     @PersistenceContext(unitName = "jpacrepo_pu")
@@ -41,70 +45,95 @@ public class PacmanServiceEJB implements PacmanServiceView
     @Resource
     private UserTransaction ut;
 
-    @Resource(lookup = "java:jboss/ee/concurrency/factory/default")
-    private ManagedThreadFactory mtf;
-
     private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName());
 
 
     private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
     private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
 
-    private Map<String, PkgData> knownPkg;
+    private Set<String> knownPkg;
 
+    @Asynchronous
     @Schedule(hour = "3", minute = "00", persistent = false)
     public void syncDB()
     {
         logger.log(Level.INFO, "Starting repository cleanup");
-        knownPkg = new HashMap<>();
+        knownPkg = new HashSet<>();
         //Elimina i pacchetti sul DB che non esistono più nel filesystem
+        logger.info("Searching for packages that are no more in the filesystem");
         try
         {
             ut.setTransactionTimeout(1000);
             ut.begin();
-            List<PkgData> listaDB = em.createQuery("SELECT p FROM PkgData p", PkgData.class).getResultList();
+            List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList();
-            for (PkgData p : listaDB)
+            logger.info("Got list of filenames from db");
+            for (String fileName : listaDB)
             {
-                knownPkg.put(p.fileName, p);
+                File file = ctx.getFile(fileName);
-                File file = ctx.getFile(p);
                 if (!file.exists())
                 {
                     logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName()));
-                    em.remove(p);
+                    PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName",
+                            PkgData.class).setParameter("fileName", file.getName()).getSingleResult();
+                    em.remove(pkg);
+                }
+                else
+                {
+                    knownPkg.add(fileName);
                 }
             }
             ut.commit();
-        } catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e)
+        }
+        catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e)
         {
             throw new RuntimeException(e);
         }
 
-        //Aggiunge sul DB i pacchetti presenti nel filesystem
+        logger.info("Searching for new packages or packages that were modified after being added to the database");
-        int cores = Runtime.getRuntime().availableProcessors();
+        List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
 
-        List<File> ls = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
+        Parser parser = new Parser();
-        Deque<File> stack = new LinkedList<>(ls);
+        for(File file : pkgfiles)
-        Thread[] works = new Thread[cores];
-        for(int i=0; i<cores; i++)
         {
-                works[i] = mtf.newThread(new SyncWorker(stack));
+
-        }
+            boolean update = !knownPkg.contains(file.getName());
-        for(Thread t : works)
+            if(!update)
-        {
-            t.start();
-        }
-        for(Thread t : works)
-        {
-            try
             {
-                t.join();
+                TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
+                query.setParameter("filename", file.getName());
+                Date result = query.getSingleResult();
+                if(file.lastModified() > result.getTime())
+                {
+                    update = true;
+                }
             }
-            catch (InterruptedException e)
+
+            if (update)
             {
-                e.printStackTrace();
+                try
+                {
+                    ut.begin();
+                    parseFile(parser, file);
+                    ut.commit();
+                }
+                catch (Exception e)
+                {
+                    logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath()));
+                    try
+                    {
+                        ut.rollback();
+                    }
+                    catch (SystemException e1)
+                    {
+                        throw new RuntimeException(e1);
+                    }
+                    throw new RuntimeException(e);
+                }
             }
         }
+        logger.log(Level.INFO, "Removing obsolete packages");
         deleteOld();
+        logger.info("Repository cleanup completed successfully");
     }
 
     private void parseFile(Parser p, File file) throws Exception
@@ -114,7 +143,7 @@ public class PacmanServiceEJB implements PacmanServiceView
 
         PkgData data = p.parseFile(file);
         hquery.setParameter("md5sum", data.md5sum);
-        List<PkgData>  savedFiles = hquery.getResultList();
+        List<PkgData> savedFiles = hquery.getResultList();
         if (savedFiles.size() > 0)
         {
             return;
@@ -124,7 +153,7 @@ public class PacmanServiceEJB implements PacmanServiceView
             TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
             fquery.setParameter("fileName", file.getName());
             savedFiles = fquery.getResultList();
-            if(savedFiles.size()>0)
+            if (savedFiles.size() > 0)
             {
                 em.remove(savedFiles.get(0));
             }
@@ -148,7 +177,7 @@ public class PacmanServiceEJB implements PacmanServiceView
         TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
         fquery.setParameter("fileName", filename);
         List<PkgData> savedFiles = fquery.getResultList();
-        if(savedFiles.size()==0)
+        if (savedFiles.size() == 0)
         {
             ut.rollback();
             throw new RuntimeException(String.format("Package with name %s not found", filename));
@@ -166,58 +195,9 @@ public class PacmanServiceEJB implements PacmanServiceView
 
         em.remove(pkg);
         ut.commit();
-        logger.log(Level.INFO,String.format("Package %s has been deleted", filename));
+        logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
     }
 
-    private class SyncWorker implements Runnable
-    {
-        Deque<File> ls;
-
-        Parser parser = new Parser();
-
-        public SyncWorker(Deque<File> ls)
-        {
-            this.ls = ls;
-        }
-
-        @Override
-        public void run()
-        {
-            while(ls.size() > 0)
-            {
-                File file;
-                synchronized (ls)
-                {
-                    file = ls.pop();
-                }
-                try
-                {
-                    PkgData p = knownPkg.get(file.getName());
-                    if (p == null || file.lastModified() > p.updTimestamp.getTime())
-                    {
-                        ut.begin();
-                        parseFile(parser, file);
-                        ut.commit();
-                    }
-                }
-                catch (Exception e)
-                {
-                    logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath()));
-                    try
-                    {
-                        ut.rollback();
-                    }
-                    catch (SystemException e1)
-                    {
-                        throw new RuntimeException(e1);
-                    }
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-    }
-
-
     private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" +
             "(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)";
 
@@ -227,10 +207,10 @@ public class PacmanServiceEJB implements PacmanServiceView
         Calendar cutoff = Calendar.getInstance();
         cutoff.add(Calendar.YEAR, -2);
         query.setParameter("cutoff", cutoff.getTime());
-        query.setParameter("minVersions", (long)2);
+        query.setParameter("minVersions", (long) 2);
         List<String> list = query.getResultList();
-        logger.log(Level.INFO, "Removing obsolete packages");
+        list.forEach((el) ->
-        list.forEach((el) -> {
+        {
             try
             {
                 deletePackage(el);
@@ -253,25 +233,25 @@ public class PacmanServiceEJB implements PacmanServiceView
         builder = em.getCriteriaBuilder();
         criteriaQuery = builder.createQuery(Long.class);
         entity = criteriaQuery.from(PkgData.class);
-        Predicate finalPredicate=null, p;
+        Predicate finalPredicate = null, p;
 
-        if(name != null && !name.isEmpty())
+        if (name != null && !name.isEmpty())
         {
             p = builder.equal(entity.get("name").get("id"), name);
             finalPredicate = p;
         }
-        if(version != null && !version.isEmpty())
+        if (version != null && !version.isEmpty())
         {
-            p=builder.equal(entity.get("version"), version);
+            p = builder.equal(entity.get("version"), version);
             finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
         }
-        if(arch != null && !arch.isEmpty())
+        if (arch != null && !arch.isEmpty())
         {
-            p=builder.equal(entity.get("arch"), arch);
+            p = builder.equal(entity.get("arch"), arch);
             finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
         }
 
-        if(finalPredicate != null)
+        if (finalPredicate != null)
         {
             criteriaQuery.select(builder.count(entity)).where(finalPredicate);
         }
@@ -284,35 +264,35 @@ public class PacmanServiceEJB implements PacmanServiceView
 
     @Override
     @Lock(LockType.READ)
-    public<T> List<T> listProperty(String property, Map<String,String> equalityConditions, Class<T> cls)
+    public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls)
     {
         CriteriaBuilder builder = em.getCriteriaBuilder();
         CriteriaQuery<T> criteriaQuery = builder.createQuery(cls);
         Root<PkgData> entity = criteriaQuery.from(PkgData.class);
 
-        Predicate finalPredicate=null, p;
+        Predicate finalPredicate = null, p;
         String key = equalityConditions.get("name");
-        if(key != null && !key.isEmpty())
+        if (key != null && !key.isEmpty())
         {
             p = builder.equal(entity.get("name").get("id"), key);
             finalPredicate = p;
         }
 
         key = equalityConditions.get("version");
-        if(key != null && !key.isEmpty())
+        if (key != null && !key.isEmpty())
         {
             p = builder.equal(entity.get("version"), key);
             finalPredicate = p;
         }
 
         key = equalityConditions.get("arch");
-        if(key != null && !key.isEmpty())
+        if (key != null && !key.isEmpty())
         {
             p = builder.equal(entity.get("arch"), key);
             finalPredicate = p;
         }
 
-        if(finalPredicate != null)
+        if (finalPredicate != null)
         {
             criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate);
         }
@@ -335,25 +315,25 @@ public class PacmanServiceEJB implements PacmanServiceView
         builder = em.getCriteriaBuilder();
         criteriaQuery = builder.createQuery(PkgData.class);
         entity = criteriaQuery.from(PkgData.class);
-        Predicate finalPredicate=null, p;
+        Predicate finalPredicate = null, p;
 
-        if(name != null && !name.isEmpty())
+        if (name != null && !name.isEmpty())
         {
             p = builder.equal(entity.get("name").get("id"), name);
             finalPredicate = p;
         }
-        if(version != null && !version.isEmpty())
+        if (version != null && !version.isEmpty())
         {
-            p=builder.equal(entity.get("version"), version);
+            p = builder.equal(entity.get("version"), version);
             finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
         }
-        if(arch != null && !arch.isEmpty())
+        if (arch != null && !arch.isEmpty())
         {
-            p=builder.equal(entity.get("arch"), arch);
+            p = builder.equal(entity.get("arch"), arch);
             finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
         }
 
-        if(finalPredicate != null)
+        if (finalPredicate != null)
         {
             criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
         }
@@ -362,11 +342,11 @@ public class PacmanServiceEJB implements PacmanServiceView
             criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
         }
         TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
-        if(pageNumber>=0)
+        if (pageNumber >= 0)
         {
-            query.setFirstResult(pageNumber*pageSize);
+            query.setFirstResult(pageNumber * pageSize);
         }
-        if(pageSize>0)
+        if (pageSize > 0)
         {
             query.setMaxResults(pageSize);
         }

src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java

@@ -9,5 +9,6 @@ import javax.ejb.Remote;
 @Remote
 public interface PacmanServiceRemote
 {
+    public void syncDB();
     public void deletePackage(String filename) throws Exception;
 }

src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java

@@ -1,11 +1,11 @@
 package com.oggio88.jpacrepo.service;
 
-import com.oggio88.jpacrepo.model.PkgName;
 import com.oggio88.jpacrepo.context.ApplicationContext;
 import com.oggio88.jpacrepo.context.DefaultConfiguration;
-import com.oggio88.jpacrepo.model.StringList;
 import com.oggio88.jpacrepo.model.PkgData;
 import com.oggio88.jpacrepo.model.PkgList;
+import com.oggio88.jpacrepo.model.PkgName;
+import com.oggio88.jpacrepo.model.StringList;
 import com.oggio88.jpacrepo.pacbase.Parser;
 import com.oggio88.jpacrepo.persistence.QueryEngine;
 
@@ -21,6 +21,8 @@ import java.io.*;
 import java.net.URI;
 import java.nio.file.Files;
 import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -71,7 +73,7 @@ public class PacmanWebService
         {
             return getPackageByFileName(filename);
         }
-        else if (name != null || arch != null || version != null)
+        else
         {
 
             QueryEngine qe = new QueryEngine(PkgData.class);
@@ -82,10 +84,70 @@ public class PacmanWebService
             String query = qe.build();
             return manageQueryResult(em.createQuery(query).getResultList());
         }
-        else
+    }
+
+    @GET
+    @Path("searchByName/{name}")
+    public Response searchPackagebyName(@PathParam("name") String name)
+    {
+        if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
+        String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name);
+        return Response.ok(em.createQuery(query, String.class).getResultList()).build();
+    }
+
+    @GET
+    @Path("list/{name}")
+    public Response getPackage(@PathParam("name") String name)
+    {
+        TypedQuery query = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", String.class);
+        query.setParameter("name", name);
+        return Response.ok(query.getResultList()).build();
+    }
+
+    @GET
+    @Path("list/{name}/{version}")
+    public Response getPackage(@PathParam("name") String name, @PathParam("version") String version)
+    {
+        TypedQuery query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", String.class);
+        query.setParameter("name", name);
+        query.setParameter("version", version);
+        return Response.ok(query.getResultList()).build();
+    }
+
+    @GET
+    @Path("list/{name}/{version}/{arch}")
+    public Response getPackage(@PathParam("name") String name, @PathParam("version") String version, @PathParam("arch") String arch)
+    {
+        TypedQuery<PkgData> query = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " +
+                "pkg.name.id = :name AND " +
+                "pkg.version = :version AND " +
+                "pkg.arch = :arch " +
+                "ORDER BY pkg.arch", PkgData.class);
+        query.setParameter("name", name);
+        query.setParameter("version", version);
+        query.setParameter("arch", arch);
+        return Response.ok(query.getSingleResult()).build();
+    }
+
+    @GET
+    @Path("map")
+    public Response getPackageMap()
+    {
+        TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class);
+        SortedMap<String, SortedMap<String, SortedMap<String, String>>> result = new TreeMap<>();
+        for (Object[] pkg : query.getResultList())
         {
-            throw new NotFoundException();
+            String name = (String) pkg[0];
+            String version = (String) pkg[1];
+            String arch = (String) pkg[2];
+            String filename = (String) pkg[3];
+            result.putIfAbsent(name, new TreeMap<>());
+            SortedMap<String, SortedMap<String, String>> map = result.get(name);
+            map.putIfAbsent(version, new TreeMap<>());
+            SortedMap<String, String> map2 = map.get(version);
+            map2.putIfAbsent(arch, filename);
         }
+        return Response.ok(result).build();
     }
 
     @GET
@@ -240,7 +302,6 @@ public class PacmanWebService
 
     private Response manageQueryResult(List<PkgData> list, boolean singleResult)
     {
-//        log.log(Level.INFO, "size: " + list.size());
         PkgList pkgList = new PkgList(list);
         if (pkgList.size() == 0)
         {

src/test/java/ClientTest.java

@@ -1,3 +1,4 @@
+import com.oggio88.jpacrepo.service.PacmanServiceRemote;
 import com.thoughtworks.xstream.XStream;
 import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
 import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
@@ -133,18 +134,21 @@ public class ClientTest
 
         prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory");
 //        prop.put(Context.PROVIDER_URL, "http-remoting://localhost:8080");
-        prop.put(Context.PROVIDER_URL, "http-remoting://odroid-xu4:8080");
+        prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
 //        prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
-        prop.put(Context.SECURITY_PRINCIPAL, "jpacrepo");
+        prop.put(Context.SECURITY_PRINCIPAL, "walter");
-        prop.put(Context.SECURITY_CREDENTIALS, "password01.");
+        prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d");
         prop.put("jboss.naming.client.ejb.context", true);
         Context context = new InitialContext(prop);
         Context ctx = new InitialContext(prop);
         traverseJndiNode("/", context);
 //        final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
-        final PacmanServiceView service = (PacmanServiceView) ctx.lookup("/jpacrepo/PacmanServiceEJB!org.jpacrepo.service.PacmanService");
+        final PacmanServiceRemote service = (PacmanServiceRemote) ctx.lookup(
-        List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
+                "/jpacrepo-2.0/PacmanServiceEJB!com.oggio88.jpacrepo.service.PacmanServiceRemote"
-        System.out.println(new XStream().toXML(pkgs));
+        );
+//        List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
+//        System.out.println(new XStream().toXML(pkgs));
+        service.syncDB();
     }
 
 }