woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

added new map service with server side caching

Browse Source
This commit is contained in:
Walter Oggioni
2017-09-19 20:43:09 +02:00
parent 671f74231a
commit 1aecc6628e
5 changed files with 146 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java
View File

@@ -11,6 +11,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.SortedMap;
/**
* Created by walter on 29/03/15.
@@ -30,6 +31,8 @@ public class ApplicationContext
private PacmanServiceView pacmanService;
public boolean invalidateCache = true;
public ApplicationContext(String propertyFile)
{
systemProperties = new Properties();
@@ -88,5 +91,4 @@ public class ApplicationContext
{
this.pacmanService = pacmanService;
}
}

39
src/main/java/com/oggio88/jpacrepo/pacbase/Parser.java
View File

@@ -16,50 +16,43 @@ import java.util.*;
*/
public class Parser
{
private XZCompressorInputStream xzcis;
private TarArchiveInputStream tais;
private ArchiveEntry ae;
private Hasher hasher;
public Parser()
{
hasher = new Hasher("MD5");
}
public PkgData parseFile(File file) throws Exception
public static PkgData parseFile(File file) throws Exception
{
XZCompressorInputStream xzcis;
TarArchiveInputStream tais;
ArchiveEntry ae;
Hasher hasher = new Hasher("MD5");
xzcis = new XZCompressorInputStream(new FileInputStream(file));
tais = new TarArchiveInputStream(xzcis);
while((ae = tais.getNextEntry()) != null)
while ((ae = tais.getNextEntry()) != null)
{
if(ae.getName().equals(".PKGINFO"))
if (ae.getName().equals(".PKGINFO"))
{
Map<String, List<String>> propMap = new HashMap<>();
byte[] buffer = new byte[(int)tais.getCurrentEntry().getSize()];
byte[] buffer = new byte[(int) tais.getCurrentEntry().getSize()];
tais.read(buffer);
String info = new String(buffer, Charset.forName("UTF8"));
for(String line : info.split("\n"))
for (String line : info.split("\n"))
{
if(line.startsWith("#") || line.trim().length() == 0)
if (line.startsWith("#") || line.trim().length() == 0)
{
continue;
}
else
{
int equals = line.indexOf("=");
if(equals < 0)
if (equals < 0)
{
throw new RuntimeException("Error parsing .PKGINFO file");
}
else
{
String key = line.substring(0, equals).trim();
if(propMap.get(key) == null)
if (propMap.get(key) == null)
{
propMap.put(key, new ArrayList<>());
}
propMap.get(key).add(line.substring(equals+1, line.length()).trim());
propMap.get(key).add(line.substring(equals + 1, line.length()).trim());
}
}
}
@@ -67,9 +60,9 @@ public class Parser
xzcis.close();
PkgData data = new PkgData();
for(String key : propMap.keySet())
for (String key : propMap.keySet())
{
switch(key)
switch (key)
{
case "size":
data.size = Long.valueOf(propMap.get(key).get(0));
@@ -90,7 +83,7 @@ public class Parser
data.name = new PkgName(propMap.get(key).get(0));
break;
case "builddate":
data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0))*1000);
data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0)) * 1000);
break;
case "license":
data.license = propMap.get(key).get(0);

51
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -11,15 +11,13 @@ import javax.annotation.Resource;
import javax.ejb.*;
import javax.enterprise.concurrent.ManagedThreadFactory;
import javax.inject.Inject;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.persistence.*;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.transaction.*;
import javax.transaction.RollbackException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -47,16 +45,14 @@ public class PacmanServiceEJB implements PacmanServiceView
private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName());
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
private Set<String> knownPkg;
@Asynchronous
@Schedule(hour = "3", minute = "00", persistent = false)
public void syncDB()
{
Set<String> knownPkg;
logger.log(Level.INFO, "Starting repository cleanup");
knownPkg = new HashSet<>();
//Elimina i pacchetti sul DB che non esistono più nel filesystem
@@ -93,16 +89,16 @@ public class PacmanServiceEJB implements PacmanServiceView
List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
Parser parser = new Parser();
for(File file : pkgfiles)
for (File file : pkgfiles)
{
boolean update = !knownPkg.contains(file.getName());
if(!update)
if (!update)
{
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getName());
Date result = query.getSingleResult();
if(file.lastModified() > result.getTime())
if (file.lastModified() > result.getTime())
{
update = true;
}
@@ -134,6 +130,7 @@ public class PacmanServiceEJB implements PacmanServiceView
logger.log(Level.INFO, "Removing obsolete packages");
deleteOld();
logger.info("Repository cleanup completed successfully");
ctx.invalidateCache = true;
}
private void parseFile(Parser p, File file) throws Exception
@@ -143,34 +140,40 @@ public class PacmanServiceEJB implements PacmanServiceView
PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum);
List<PkgData> savedFiles = hquery.getResultList();
if (savedFiles.size() > 0)
try
{
hquery.getSingleResult();
return;
}
else
catch (NoResultException e)
{
}
try
{
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", file.getName());
savedFiles = fquery.getResultList();
if (savedFiles.size() > 0)
{
em.remove(savedFiles.get(0));
}
PkgData savedFile = fquery.getSingleResult();
em.remove(savedFile);
em.flush();
}
catch (NoResultException e)
{
}
nquery.setParameter("name", data.name.id);
List<PkgName> savedName = nquery.getResultList();
if (savedName.size() > 0)
try
{
PkgName savedName = nquery.getSingleResult();
data.name = savedName;
}
catch (NoResultException e)
{
data.name = savedName.get(0);
}
em.persist(data);
logger.log(Level.INFO, String.format("Persisting package %s", file.getName()));
}
@Override
@Lock(LockType.READ)
public void deletePackage(String filename) throws Exception
{
ut.begin();
@@ -223,7 +226,6 @@ public class PacmanServiceEJB implements PacmanServiceView
}
@Override
@Lock(LockType.READ)
public long countResults(String name, String version, String arch)
{
CriteriaBuilder builder;
@@ -263,7 +265,6 @@ public class PacmanServiceEJB implements PacmanServiceView
}
@Override
@Lock(LockType.READ)
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls)
{
CriteriaBuilder builder = em.getCriteriaBuilder();
@@ -305,7 +306,6 @@ public class PacmanServiceEJB implements PacmanServiceView
@Override
@Lock(LockType.READ)
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize)
{
CriteriaBuilder builder;
@@ -351,6 +351,5 @@ public class PacmanServiceEJB implements PacmanServiceView
query.setMaxResults(pageSize);
}
return query.getResultList();
}
}

121
src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java
View File

@@ -13,6 +13,10 @@ import javax.annotation.Resource;
import javax.ejb.*;
import javax.inject.Inject;
import javax.persistence.*;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.transaction.Status;
import javax.transaction.UserTransaction;
import javax.ws.rs.*;
@@ -26,15 +30,15 @@ import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@ConcurrencyManagement(ConcurrencyManagementType.BEAN)
@AccessTimeout(-1)
@Singleton
@Path("/pkg")
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Singleton
@TransactionManagement(TransactionManagementType.BEAN)
public class PacmanWebService
{
private SortedMap<String, SortedMap<String, SortedMap<String, String>>> cachedMap = null;
@PersistenceContext(unitName = "jpacrepo_pu")
private EntityManager em;
@@ -55,8 +59,6 @@ public class PacmanWebService
@DefaultConfiguration
private ApplicationContext ctx;
private final Parser serviceParser = new Parser();
@GET
@Path("search")
public Response getPackage(@QueryParam("name") String name,
@@ -88,13 +90,20 @@ public class PacmanWebService
@GET
@Path("searchByName/{name}")
public Response searchPackagebyName(@PathParam("name") String name)
public Response searchByName(@PathParam("name") String name)
{
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name);
return Response.ok(em.createQuery(query, String.class).getResultList()).build();
}
@GET
@Path("searchByHash/{md5sum}")
public Response searchByHash(@PathParam("md5sum") String md5sum)
{
return getPackageByHash(md5sum);
}
@GET
@Path("list/{name}")
public Response getPackage(@PathParam("name") String name)
@@ -133,19 +142,29 @@ public class PacmanWebService
@Path("map")
public Response getPackageMap()
{
TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class);
SortedMap<String, SortedMap<String, SortedMap<String, String>>> result = new TreeMap<>();
for (Object[] pkg : query.getResultList())
SortedMap<String, SortedMap<String, SortedMap<String, String>>> result;
if (ctx.invalidateCache)
{
String name = (String) pkg[0];
String version = (String) pkg[1];
String arch = (String) pkg[2];
String filename = (String) pkg[3];
result.putIfAbsent(name, new TreeMap<>());
SortedMap<String, SortedMap<String, String>> map = result.get(name);
map.putIfAbsent(version, new TreeMap<>());
SortedMap<String, String> map2 = map.get(version);
map2.putIfAbsent(arch, filename);
result = new TreeMap<>();
TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class);
for (Object[] pkg : query.getResultList())
{
String name = (String) pkg[0];
String version = (String) pkg[1];
String arch = (String) pkg[2];
String filename = (String) pkg[3];
result.putIfAbsent(name, new TreeMap<>());
SortedMap<String, SortedMap<String, String>> map = result.get(name);
map.putIfAbsent(version, new TreeMap<>());
SortedMap<String, String> map2 = map.get(version);
map2.putIfAbsent(arch, filename);
}
cachedMap = result;
ctx.invalidateCache = false;
}
else
{
result = cachedMap;
}
return Response.ok(result).build();
}
@@ -266,7 +285,7 @@ public class PacmanWebService
fos.close();
ut.begin();
PkgData pkg = serviceParser.parseFile(file);
PkgData pkg = Parser.parseFile(file);
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
nquery.setParameter("name", pkg.name.id);
@@ -280,6 +299,8 @@ public class PacmanWebService
log.log(Level.INFO, String.format("Persisting package %s", pkg.fileName));
URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.fileName).build();
ut.commit();
ctx.invalidateCache = true;
cachedMap = null;
return Response.created(pkgUri).build();
}
catch (Exception e)
@@ -295,6 +316,67 @@ public class PacmanWebService
}
}
@GET
@Path("/search")
public List<PkgData> searchPackage(
@QueryParam("name") String name,
@QueryParam("version") String version,
@QueryParam("arch") String arch,
@QueryParam("page") int pageNumber,
@QueryParam("pageSize") int pageSize,
@QueryParam("fileName") String fileName
)
{
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(PkgData.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (fileName != null && !fileName.isEmpty())
{
p = builder.equal(entity.get("fileName"), fileName);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
else
{
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0)
{
query.setFirstResult(pageNumber * pageSize);
}
if (pageSize > 0)
{
query.setMaxResults(pageSize);
}
return query.getResultList();
}
private Response manageQueryResult(List<PkgData> list)
{
return manageQueryResult(list, false);
@@ -323,5 +405,4 @@ public class PacmanWebService
return Response.ok(pkgList).build();
}
}
}

2
src/test/java/ClientTest.java
View File

@@ -151,4 +151,4 @@ public class ClientTest
service.syncDB();
}
}
}