woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

added new map service with server side caching

Browse Source
This commit is contained in:
Walter Oggioni
2017-09-19 20:43:09 +02:00
parent 671f74231a
commit 1aecc6628e
5 changed files with 146 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java
View File

@@ -11,6 +11,7 @@ import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import java.util.SortedMap;
/** /**
* Created by walter on 29/03/15. * Created by walter on 29/03/15.
@@ -30,6 +31,8 @@ public class ApplicationContext
private PacmanServiceView pacmanService; private PacmanServiceView pacmanService;
public boolean invalidateCache = true;
public ApplicationContext(String propertyFile) public ApplicationContext(String propertyFile)
{ {
systemProperties = new Properties(); systemProperties = new Properties();
@@ -88,5 +91,4 @@ public class ApplicationContext
{ {
this.pacmanService = pacmanService; this.pacmanService = pacmanService;
} }
} }

39
src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java
View File

@@ -16,50 +16,43 @@ import java.util.*;
*/ */
public class Parser public class Parser
{ {
public static PkgData parseFile(File file) throws Exception
private XZCompressorInputStream xzcis;
private TarArchiveInputStream tais;
private ArchiveEntry ae;
private Hasher hasher;
public Parser()
{
hasher = new Hasher("MD5");
}
public PkgData parseFile(File file) throws Exception
{ {
XZCompressorInputStream xzcis;
TarArchiveInputStream tais;
ArchiveEntry ae;
Hasher hasher = new Hasher("MD5");
xzcis = new XZCompressorInputStream(new FileInputStream(file)); xzcis = new XZCompressorInputStream(new FileInputStream(file));
tais = new TarArchiveInputStream(xzcis); tais = new TarArchiveInputStream(xzcis);
while((ae = tais.getNextEntry()) != null) while ((ae = tais.getNextEntry()) != null)
{ {
if(ae.getName().equals(".PKGINFO")) if (ae.getName().equals(".PKGINFO"))
{ {
Map<String, List<String>> propMap = new HashMap<>(); Map<String, List<String>> propMap = new HashMap<>();
byte[] buffer = new byte[(int)tais.getCurrentEntry().getSize()]; byte[] buffer = new byte[(int) tais.getCurrentEntry().getSize()];
tais.read(buffer); tais.read(buffer);
String info = new String(buffer, Charset.forName("UTF8")); String info = new String(buffer, Charset.forName("UTF8"));
for(String line : info.split("\n")) for (String line : info.split("\n"))
{ {
if(line.startsWith("#") || line.trim().length() == 0) if (line.startsWith("#") || line.trim().length() == 0)
{ {
continue; continue;
} }
else else
{ {
int equals = line.indexOf("="); int equals = line.indexOf("=");
if(equals < 0) if (equals < 0)
{ {
throw new RuntimeException("Error parsing .PKGINFO file"); throw new RuntimeException("Error parsing .PKGINFO file");
} }
else else
{ {
String key = line.substring(0, equals).trim(); String key = line.substring(0, equals).trim();
if(propMap.get(key) == null) if (propMap.get(key) == null)
{ {
propMap.put(key, new ArrayList<>()); propMap.put(key, new ArrayList<>());
} }
propMap.get(key).add(line.substring(equals+1, line.length()).trim()); propMap.get(key).add(line.substring(equals + 1, line.length()).trim());
} }
} }
} }
@@ -67,9 +60,9 @@ public class Parser
xzcis.close(); xzcis.close();
PkgData data = new PkgData(); PkgData data = new PkgData();
for(String key : propMap.keySet()) for (String key : propMap.keySet())
{ {
switch(key) switch (key)
{ {
case "size": case "size":
data.size = Long.valueOf(propMap.get(key).get(0)); data.size = Long.valueOf(propMap.get(key).get(0));
@@ -90,7 +83,7 @@ public class Parser
data.name = new PkgName(propMap.get(key).get(0)); data.name = new PkgName(propMap.get(key).get(0));
break; break;
case "builddate": case "builddate":
data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0))*1000); data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0)) * 1000);
break; break;
case "license": case "license":
data.license = propMap.get(key).get(0); data.license = propMap.get(key).get(0);

49
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -11,15 +11,13 @@ import javax.annotation.Resource;
import javax.ejb.*; import javax.ejb.*;
import javax.enterprise.concurrent.ManagedThreadFactory; import javax.enterprise.concurrent.ManagedThreadFactory;
import javax.inject.Inject; import javax.inject.Inject;
import javax.persistence.EntityManager; import javax.persistence.*;
import javax.persistence.PersistenceContext;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder; import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root; import javax.persistence.criteria.Root;
import javax.transaction.*; import javax.transaction.*;
import javax.transaction.RollbackException;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
@@ -47,16 +45,14 @@ public class PacmanServiceEJB implements PacmanServiceView
private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName()); private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName());
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name"; private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"; private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
private Set<String> knownPkg;
@Asynchronous @Asynchronous
@Schedule(hour = "3", minute = "00", persistent = false) @Schedule(hour = "3", minute = "00", persistent = false)
public void syncDB() public void syncDB()
{ {
Set<String> knownPkg;
logger.log(Level.INFO, "Starting repository cleanup"); logger.log(Level.INFO, "Starting repository cleanup");
knownPkg = new HashSet<>(); knownPkg = new HashSet<>();
//Elimina i pacchetti sul DB che non esistono più nel filesystem //Elimina i pacchetti sul DB che non esistono più nel filesystem
@@ -93,16 +89,16 @@ public class PacmanServiceEJB implements PacmanServiceView
List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz")))); List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
Parser parser = new Parser(); Parser parser = new Parser();
for(File file : pkgfiles) for (File file : pkgfiles)
{ {
boolean update = !knownPkg.contains(file.getName()); boolean update = !knownPkg.contains(file.getName());
if(!update) if (!update)
{ {
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class); TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getName()); query.setParameter("filename", file.getName());
Date result = query.getSingleResult(); Date result = query.getSingleResult();
if(file.lastModified() > result.getTime()) if (file.lastModified() > result.getTime())
{ {
update = true; update = true;
} }
@@ -134,6 +130,7 @@ public class PacmanServiceEJB implements PacmanServiceView
logger.log(Level.INFO, "Removing obsolete packages"); logger.log(Level.INFO, "Removing obsolete packages");
deleteOld(); deleteOld();
logger.info("Repository cleanup completed successfully"); logger.info("Repository cleanup completed successfully");
ctx.invalidateCache = true;
} }
private void parseFile(Parser p, File file) throws Exception private void parseFile(Parser p, File file) throws Exception
@@ -143,34 +140,40 @@ public class PacmanServiceEJB implements PacmanServiceView
PkgData data = p.parseFile(file); PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum); hquery.setParameter("md5sum", data.md5sum);
List<PkgData> savedFiles = hquery.getResultList(); try
if (savedFiles.size() > 0)
{ {
hquery.getSingleResult();
return; return;
} }
else catch (NoResultException e)
{
}
try
{ {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class); TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", file.getName()); fquery.setParameter("fileName", file.getName());
savedFiles = fquery.getResultList(); PkgData savedFile = fquery.getSingleResult();
if (savedFiles.size() > 0) em.remove(savedFile);
{ em.flush();
em.remove(savedFiles.get(0));
} }
catch (NoResultException e)
{
} }
nquery.setParameter("name", data.name.id); nquery.setParameter("name", data.name.id);
List<PkgName> savedName = nquery.getResultList(); try
if (savedName.size() > 0) {
PkgName savedName = nquery.getSingleResult();
data.name = savedName;
}
catch (NoResultException e)
{ {
data.name = savedName.get(0);
} }
em.persist(data); em.persist(data);
logger.log(Level.INFO, String.format("Persisting package %s", file.getName())); logger.log(Level.INFO, String.format("Persisting package %s", file.getName()));
} }
@Override @Override
@Lock(LockType.READ)
public void deletePackage(String filename) throws Exception public void deletePackage(String filename) throws Exception
{ {
ut.begin(); ut.begin();
@@ -223,7 +226,6 @@ public class PacmanServiceEJB implements PacmanServiceView
} }
@Override @Override
@Lock(LockType.READ)
public long countResults(String name, String version, String arch) public long countResults(String name, String version, String arch)
{ {
CriteriaBuilder builder; CriteriaBuilder builder;
@@ -263,7 +265,6 @@ public class PacmanServiceEJB implements PacmanServiceView
} }
@Override @Override
@Lock(LockType.READ)
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls) public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls)
{ {
CriteriaBuilder builder = em.getCriteriaBuilder(); CriteriaBuilder builder = em.getCriteriaBuilder();
@@ -305,7 +306,6 @@ public class PacmanServiceEJB implements PacmanServiceView
@Override @Override
@Lock(LockType.READ)
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize) public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize)
{ {
CriteriaBuilder builder; CriteriaBuilder builder;
@@ -351,6 +351,5 @@ public class PacmanServiceEJB implements PacmanServiceView
query.setMaxResults(pageSize); query.setMaxResults(pageSize);
} }
return query.getResultList(); return query.getResultList();
} }
} }

99
src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java
View File

@@ -13,6 +13,10 @@ import javax.annotation.Resource;
import javax.ejb.*; import javax.ejb.*;
import javax.inject.Inject; import javax.inject.Inject;
import javax.persistence.*; import javax.persistence.*;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.transaction.Status; import javax.transaction.Status;
import javax.transaction.UserTransaction; import javax.transaction.UserTransaction;
import javax.ws.rs.*; import javax.ws.rs.*;
@@ -26,15 +30,15 @@ import java.util.TreeMap;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
@ConcurrencyManagement(ConcurrencyManagementType.BEAN) @Singleton
@AccessTimeout(-1)
@Path("/pkg") @Path("/pkg")
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) @Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Singleton
@TransactionManagement(TransactionManagementType.BEAN) @TransactionManagement(TransactionManagementType.BEAN)
public class PacmanWebService public class PacmanWebService
{ {
private SortedMap<String, SortedMap<String, SortedMap<String, String>>> cachedMap = null;
@PersistenceContext(unitName = "jpacrepo_pu") @PersistenceContext(unitName = "jpacrepo_pu")
private EntityManager em; private EntityManager em;
@@ -55,8 +59,6 @@ public class PacmanWebService
@DefaultConfiguration @DefaultConfiguration
private ApplicationContext ctx; private ApplicationContext ctx;
private final Parser serviceParser = new Parser();
@GET @GET
@Path("search") @Path("search")
public Response getPackage(@QueryParam("name") String name, public Response getPackage(@QueryParam("name") String name,
@@ -88,13 +90,20 @@ public class PacmanWebService
@GET @GET
@Path("searchByName/{name}") @Path("searchByName/{name}")
public Response searchPackagebyName(@PathParam("name") String name) public Response searchByName(@PathParam("name") String name)
{ {
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST); if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name); String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name);
return Response.ok(em.createQuery(query, String.class).getResultList()).build(); return Response.ok(em.createQuery(query, String.class).getResultList()).build();
} }
@GET
@Path("searchByHash/{md5sum}")
public Response searchByHash(@PathParam("md5sum") String md5sum)
{
return getPackageByHash(md5sum);
}
@GET @GET
@Path("list/{name}") @Path("list/{name}")
public Response getPackage(@PathParam("name") String name) public Response getPackage(@PathParam("name") String name)
@@ -133,8 +142,11 @@ public class PacmanWebService
@Path("map") @Path("map")
public Response getPackageMap() public Response getPackageMap()
{ {
SortedMap<String, SortedMap<String, SortedMap<String, String>>> result;
if (ctx.invalidateCache)
{
result = new TreeMap<>();
TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class); TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class);
SortedMap<String, SortedMap<String, SortedMap<String, String>>> result = new TreeMap<>();
for (Object[] pkg : query.getResultList()) for (Object[] pkg : query.getResultList())
{ {
String name = (String) pkg[0]; String name = (String) pkg[0];
@@ -147,6 +159,13 @@ public class PacmanWebService
SortedMap<String, String> map2 = map.get(version); SortedMap<String, String> map2 = map.get(version);
map2.putIfAbsent(arch, filename); map2.putIfAbsent(arch, filename);
} }
cachedMap = result;
ctx.invalidateCache = false;
}
else
{
result = cachedMap;
}
return Response.ok(result).build(); return Response.ok(result).build();
} }
@@ -266,7 +285,7 @@ public class PacmanWebService
fos.close(); fos.close();
ut.begin(); ut.begin();
PkgData pkg = serviceParser.parseFile(file); PkgData pkg = Parser.parseFile(file);
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class); TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
nquery.setParameter("name", pkg.name.id); nquery.setParameter("name", pkg.name.id);
@@ -280,6 +299,8 @@ public class PacmanWebService
log.log(Level.INFO, String.format("Persisting package %s", pkg.fileName)); log.log(Level.INFO, String.format("Persisting package %s", pkg.fileName));
URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.fileName).build(); URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.fileName).build();
ut.commit(); ut.commit();
ctx.invalidateCache = true;
cachedMap = null;
return Response.created(pkgUri).build(); return Response.created(pkgUri).build();
} }
catch (Exception e) catch (Exception e)
@@ -295,6 +316,67 @@ public class PacmanWebService
} }
} }
@GET
@Path("/search")
public List<PkgData> searchPackage(
@QueryParam("name") String name,
@QueryParam("version") String version,
@QueryParam("arch") String arch,
@QueryParam("page") int pageNumber,
@QueryParam("pageSize") int pageSize,
@QueryParam("fileName") String fileName
)
{
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(PkgData.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (fileName != null && !fileName.isEmpty())
{
p = builder.equal(entity.get("fileName"), fileName);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
else
{
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0)
{
query.setFirstResult(pageNumber * pageSize);
}
if (pageSize > 0)
{
query.setMaxResults(pageSize);
}
return query.getResultList();
}
private Response manageQueryResult(List<PkgData> list) private Response manageQueryResult(List<PkgData> list)
{ {
return manageQueryResult(list, false); return manageQueryResult(list, false);
@@ -323,5 +405,4 @@ public class PacmanWebService
return Response.ok(pkgList).build(); return Response.ok(pkgList).build();
} }
} }
} }