From 1aecc6628e922077b0b5bde5b3d440941a019da9 Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Tue, 19 Sep 2017 20:43:09 +0200 Subject: [PATCH] added new map service with server side caching --- .../jpacrepo/context/ApplicationContext.java | 4 +- .../com/oggio88/jpacrepo/pacbase/Parser.java | 39 +++--- .../jpacrepo/service/PacmanServiceEJB.java | 51 ++++---- .../jpacrepo/service/PacmanWebService.java | 121 +++++++++++++++--- src/test/java/ClientTest.java | 2 +- 5 files changed, 146 insertions(+), 71 deletions(-) diff --git a/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java b/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java index 44f24ae..702c158 100644 --- a/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java +++ b/src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java @@ -11,6 +11,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Properties; +import java.util.SortedMap; /** * Created by walter on 29/03/15. @@ -30,6 +31,8 @@ public class ApplicationContext private PacmanServiceView pacmanService; + public boolean invalidateCache = true; + public ApplicationContext(String propertyFile) { systemProperties = new Properties(); @@ -88,5 +91,4 @@ public class ApplicationContext { this.pacmanService = pacmanService; } - } diff --git a/src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java b/src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java index 5adca94..5523504 100644 --- a/src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java +++ b/src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java @@ -16,50 +16,43 @@ import java.util.*; */ public class Parser { - - private XZCompressorInputStream xzcis; - private TarArchiveInputStream tais; - private ArchiveEntry ae; - private Hasher hasher; - - public Parser() - { - hasher = new Hasher("MD5"); - } - - public PkgData parseFile(File file) throws Exception + public static PkgData parseFile(File file) throws Exception { + XZCompressorInputStream xzcis; + TarArchiveInputStream tais; + ArchiveEntry ae; + Hasher hasher = new Hasher("MD5"); xzcis = new XZCompressorInputStream(new FileInputStream(file)); tais = new TarArchiveInputStream(xzcis); - while((ae = tais.getNextEntry()) != null) + while ((ae = tais.getNextEntry()) != null) { - if(ae.getName().equals(".PKGINFO")) + if (ae.getName().equals(".PKGINFO")) { Map> propMap = new HashMap<>(); - byte[] buffer = new byte[(int)tais.getCurrentEntry().getSize()]; + byte[] buffer = new byte[(int) tais.getCurrentEntry().getSize()]; tais.read(buffer); String info = new String(buffer, Charset.forName("UTF8")); - for(String line : info.split("\n")) + for (String line : info.split("\n")) { - if(line.startsWith("#") || line.trim().length() == 0) + if (line.startsWith("#") || line.trim().length() == 0) { continue; } else { int equals = line.indexOf("="); - if(equals < 0) + if (equals < 0) { throw new RuntimeException("Error parsing .PKGINFO file"); } else { String key = line.substring(0, equals).trim(); - if(propMap.get(key) == null) + if (propMap.get(key) == null) { propMap.put(key, new ArrayList<>()); } - propMap.get(key).add(line.substring(equals+1, line.length()).trim()); + propMap.get(key).add(line.substring(equals + 1, line.length()).trim()); } } } @@ -67,9 +60,9 @@ public class Parser xzcis.close(); PkgData data = new PkgData(); - for(String key : propMap.keySet()) + for (String key : propMap.keySet()) { - switch(key) + switch (key) { case "size": data.size = Long.valueOf(propMap.get(key).get(0)); @@ -90,7 +83,7 @@ public class Parser data.name = new PkgName(propMap.get(key).get(0)); break; case "builddate": - data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0))*1000); + data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0)) * 1000); break; case "license": data.license = propMap.get(key).get(0); diff --git a/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java b/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java index 00145ce..536bae1 100644 --- a/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java +++ b/src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java @@ -11,15 +11,13 @@ import javax.annotation.Resource; import javax.ejb.*; import javax.enterprise.concurrent.ManagedThreadFactory; import javax.inject.Inject; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.Query; -import javax.persistence.TypedQuery; +import javax.persistence.*; import javax.persistence.criteria.CriteriaBuilder; import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; import javax.transaction.*; +import javax.transaction.RollbackException; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -47,16 +45,14 @@ public class PacmanServiceEJB implements PacmanServiceView private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName()); - private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name"; private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"; - private Set knownPkg; - @Asynchronous @Schedule(hour = "3", minute = "00", persistent = false) public void syncDB() { + Set knownPkg; logger.log(Level.INFO, "Starting repository cleanup"); knownPkg = new HashSet<>(); //Elimina i pacchetti sul DB che non esistono più nel filesystem @@ -93,16 +89,16 @@ public class PacmanServiceEJB implements PacmanServiceView List pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz")))); Parser parser = new Parser(); - for(File file : pkgfiles) + for (File file : pkgfiles) { boolean update = !knownPkg.contains(file.getName()); - if(!update) + if (!update) { TypedQuery query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class); query.setParameter("filename", file.getName()); Date result = query.getSingleResult(); - if(file.lastModified() > result.getTime()) + if (file.lastModified() > result.getTime()) { update = true; } @@ -134,6 +130,7 @@ public class PacmanServiceEJB implements PacmanServiceView logger.log(Level.INFO, "Removing obsolete packages"); deleteOld(); logger.info("Repository cleanup completed successfully"); + ctx.invalidateCache = true; } private void parseFile(Parser p, File file) throws Exception @@ -143,34 +140,40 @@ public class PacmanServiceEJB implements PacmanServiceView PkgData data = p.parseFile(file); hquery.setParameter("md5sum", data.md5sum); - List savedFiles = hquery.getResultList(); - if (savedFiles.size() > 0) + try { + hquery.getSingleResult(); return; } - else + catch (NoResultException e) + { + } + try { TypedQuery fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class); fquery.setParameter("fileName", file.getName()); - savedFiles = fquery.getResultList(); - if (savedFiles.size() > 0) - { - em.remove(savedFiles.get(0)); - } + PkgData savedFile = fquery.getSingleResult(); + em.remove(savedFile); + em.flush(); + } + catch (NoResultException e) + { } nquery.setParameter("name", data.name.id); - List savedName = nquery.getResultList(); - if (savedName.size() > 0) + try + { + PkgName savedName = nquery.getSingleResult(); + data.name = savedName; + } + catch (NoResultException e) { - data.name = savedName.get(0); } em.persist(data); logger.log(Level.INFO, String.format("Persisting package %s", file.getName())); } @Override - @Lock(LockType.READ) public void deletePackage(String filename) throws Exception { ut.begin(); @@ -223,7 +226,6 @@ public class PacmanServiceEJB implements PacmanServiceView } @Override - @Lock(LockType.READ) public long countResults(String name, String version, String arch) { CriteriaBuilder builder; @@ -263,7 +265,6 @@ public class PacmanServiceEJB implements PacmanServiceView } @Override - @Lock(LockType.READ) public List listProperty(String property, Map equalityConditions, Class cls) { CriteriaBuilder builder = em.getCriteriaBuilder(); @@ -305,7 +306,6 @@ public class PacmanServiceEJB implements PacmanServiceView @Override - @Lock(LockType.READ) public List searchPackage(String name, String version, String arch, int pageNumber, int pageSize) { CriteriaBuilder builder; @@ -351,6 +351,5 @@ public class PacmanServiceEJB implements PacmanServiceView query.setMaxResults(pageSize); } return query.getResultList(); - } } \ No newline at end of file diff --git a/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java b/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java index 82491b9..0ef5278 100644 --- a/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java +++ b/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java @@ -13,6 +13,10 @@ import javax.annotation.Resource; import javax.ejb.*; import javax.inject.Inject; import javax.persistence.*; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; import javax.transaction.Status; import javax.transaction.UserTransaction; import javax.ws.rs.*; @@ -26,15 +30,15 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -@ConcurrencyManagement(ConcurrencyManagementType.BEAN) -@AccessTimeout(-1) +@Singleton @Path("/pkg") @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) @Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) -@Singleton @TransactionManagement(TransactionManagementType.BEAN) public class PacmanWebService { + private SortedMap>> cachedMap = null; + @PersistenceContext(unitName = "jpacrepo_pu") private EntityManager em; @@ -55,8 +59,6 @@ public class PacmanWebService @DefaultConfiguration private ApplicationContext ctx; - private final Parser serviceParser = new Parser(); - @GET @Path("search") public Response getPackage(@QueryParam("name") String name, @@ -88,13 +90,20 @@ public class PacmanWebService @GET @Path("searchByName/{name}") - public Response searchPackagebyName(@PathParam("name") String name) + public Response searchByName(@PathParam("name") String name) { if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST); String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name); return Response.ok(em.createQuery(query, String.class).getResultList()).build(); } + @GET + @Path("searchByHash/{md5sum}") + public Response searchByHash(@PathParam("md5sum") String md5sum) + { + return getPackageByHash(md5sum); + } + @GET @Path("list/{name}") public Response getPackage(@PathParam("name") String name) @@ -133,19 +142,29 @@ public class PacmanWebService @Path("map") public Response getPackageMap() { - TypedQuery query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class); - SortedMap>> result = new TreeMap<>(); - for (Object[] pkg : query.getResultList()) + SortedMap>> result; + if (ctx.invalidateCache) { - String name = (String) pkg[0]; - String version = (String) pkg[1]; - String arch = (String) pkg[2]; - String filename = (String) pkg[3]; - result.putIfAbsent(name, new TreeMap<>()); - SortedMap> map = result.get(name); - map.putIfAbsent(version, new TreeMap<>()); - SortedMap map2 = map.get(version); - map2.putIfAbsent(arch, filename); + result = new TreeMap<>(); + TypedQuery query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class); + for (Object[] pkg : query.getResultList()) + { + String name = (String) pkg[0]; + String version = (String) pkg[1]; + String arch = (String) pkg[2]; + String filename = (String) pkg[3]; + result.putIfAbsent(name, new TreeMap<>()); + SortedMap> map = result.get(name); + map.putIfAbsent(version, new TreeMap<>()); + SortedMap map2 = map.get(version); + map2.putIfAbsent(arch, filename); + } + cachedMap = result; + ctx.invalidateCache = false; + } + else + { + result = cachedMap; } return Response.ok(result).build(); } @@ -266,7 +285,7 @@ public class PacmanWebService fos.close(); ut.begin(); - PkgData pkg = serviceParser.parseFile(file); + PkgData pkg = Parser.parseFile(file); TypedQuery nquery = em.createQuery(nameQuery, PkgName.class); nquery.setParameter("name", pkg.name.id); @@ -280,6 +299,8 @@ public class PacmanWebService log.log(Level.INFO, String.format("Persisting package %s", pkg.fileName)); URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.fileName).build(); ut.commit(); + ctx.invalidateCache = true; + cachedMap = null; return Response.created(pkgUri).build(); } catch (Exception e) @@ -295,6 +316,67 @@ public class PacmanWebService } } + @GET + @Path("/search") + public List searchPackage( + @QueryParam("name") String name, + @QueryParam("version") String version, + @QueryParam("arch") String arch, + @QueryParam("page") int pageNumber, + @QueryParam("pageSize") int pageSize, + @QueryParam("fileName") String fileName + ) + { + CriteriaBuilder builder; + CriteriaQuery criteriaQuery; + Root entity; + + builder = em.getCriteriaBuilder(); + criteriaQuery = builder.createQuery(PkgData.class); + entity = criteriaQuery.from(PkgData.class); + Predicate finalPredicate = null, p; + + if (name != null && !name.isEmpty()) + { + p = builder.equal(entity.get("name").get("id"), name); + finalPredicate = p; + } + if (version != null && !version.isEmpty()) + { + p = builder.equal(entity.get("version"), version); + finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; + } + if (arch != null && !arch.isEmpty()) + { + p = builder.equal(entity.get("arch"), arch); + finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; + } + if (fileName != null && !fileName.isEmpty()) + { + p = builder.equal(entity.get("fileName"), fileName); + finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; + } + + if (finalPredicate != null) + { + criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName"))); + } + else + { + criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName"))); + } + TypedQuery query = em.createQuery(criteriaQuery); + if (pageNumber >= 0) + { + query.setFirstResult(pageNumber * pageSize); + } + if (pageSize > 0) + { + query.setMaxResults(pageSize); + } + return query.getResultList(); + } + private Response manageQueryResult(List list) { return manageQueryResult(list, false); @@ -323,5 +405,4 @@ public class PacmanWebService return Response.ok(pkgList).build(); } } - } \ No newline at end of file diff --git a/src/test/java/ClientTest.java b/src/test/java/ClientTest.java index 6984890..c319533 100644 --- a/src/test/java/ClientTest.java +++ b/src/test/java/ClientTest.java @@ -151,4 +151,4 @@ public class ClientTest service.syncDB(); } -} +} \ No newline at end of file