woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

added map webservice

Browse Source 
fixed syncDB EJB method
This commit is contained in:
Walter Oggioni
2017-09-06 23:32:10 +02:00
parent bc9413b1cb
commit 671f74231a
7 changed files with 200 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
build.sbt
View File

@@ -9,7 +9,7 @@ scalaVersion := "2.12.2"
libraryDependencies += "org.tukaani" % "xz" % "1.6"
libraryDependencies += "org.apache.commons" % "commons-compress" % "1.14"
libraryDependencies += "org.projectlombok" % "lombok" % "1.16.18" % "provided"
libraryDependencies += "org.projectlombok" % "lombok" % "1.16.18"
libraryDependencies += "javax" % "javaee-api" % "7.0" % "provided"
//libraryDependencies += "javax" % "javaee-api" % "7.0 "
@@ -42,6 +42,8 @@ artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
artifact.name + "-" + module.revision + "." + artifact.extension
}
//javacOptions += "-g"
val wildflyUsername = SettingKey[String]("wildfly-username", "The account username to use to connect to wildfly with jboss-cli.sh")
val wildflyPassword = SettingKey[String]("wildfly-password", "The account password to use to connect to wildfly with jboss-cli.sh")
val wildflyURL = SettingKey[String]("wildfly-url", "The username to use to connect to wildfly with jboss-cli.sh")

26
src/main/java/com/oggio88/jpacrepo/context/ApplicationContext.java
View File

@@ -28,7 +28,6 @@ public class ApplicationContext
private String repoFolder;
@Getter @Setter
private PacmanServiceView pacmanService;
public ApplicationContext(String propertyFile)
@@ -44,17 +43,20 @@ public class ApplicationContext
// get the property value and print it out
systemProperties.getProperty("RepoFolder");
repoFolder = systemProperties.getProperty("RepoFolder");
} catch (IOException ex)
}
catch (IOException ex)
{
throw new RuntimeException(ex);
} finally
}
finally
{
if (input != null)
{
try
{
input.close();
} catch (IOException e)
}
catch (IOException e)
{
throw new RuntimeException(e);
}
@@ -71,4 +73,20 @@ public class ApplicationContext
{
return new File(new File(repoFolder), pkg.fileName);
}
public File getFile(String fileName)
{
return new File(new File(repoFolder), fileName);
}
public PacmanServiceView getPacmanService()
{
return pacmanService;
}
public void setPacmanService(PacmanServiceView pacmanService)
{
this.pacmanService = pacmanService;
}
}

11
src/main/java/com/oggio88/jpacrepo/persistence/QueryEngine.java
View File

@@ -28,7 +28,7 @@ public class QueryEngine
public QueryEngine select(String... fields)
{
String[] strarr = new String[fields.length];
for(int i=0; i<fields.length; i++)
for (int i = 0; i < fields.length; i++)
{
strarr[i] = "e." + fields[i];
}
@@ -50,6 +50,13 @@ public class QueryEngine
public String build()
{
return query + " WHERE " + String.join(" AND ", where);
if (where.isEmpty())
{
return query;
}
else
{
return query + " WHERE " + String.join(" AND ", where);
}
}
}

196
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -6,12 +6,14 @@ import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.pacbase.Parser;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.ejb.*;
import javax.enterprise.concurrent.ManagedThreadFactory;
import javax.inject.Inject;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
@@ -29,6 +31,8 @@ import java.util.logging.Logger;
@Singleton
@TransactionManagement(TransactionManagementType.BEAN)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local(PacmanServiceView.class)
@Remote(PacmanServiceRemote.class)
public class PacmanServiceEJB implements PacmanServiceView
{
@PersistenceContext(unitName = "jpacrepo_pu")
@@ -41,70 +45,95 @@ public class PacmanServiceEJB implements PacmanServiceView
@Resource
private UserTransaction ut;
@Resource(lookup = "java:jboss/ee/concurrency/factory/default")
private ManagedThreadFactory mtf;
private Logger logger = Logger.getLogger(PacmanServiceEJB.class.getName());
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
private Map<String, PkgData> knownPkg;
private Set<String> knownPkg;
@Asynchronous
@Schedule(hour = "3", minute = "00", persistent = false)
public void syncDB()
{
logger.log(Level.INFO, "Starting repository cleanup");
knownPkg = new HashMap<>();
knownPkg = new HashSet<>();
//Elimina i pacchetti sul DB che non esistono più nel filesystem
logger.info("Searching for packages that are no more in the filesystem");
try
{
ut.setTransactionTimeout(1000);
ut.begin();
List<PkgData> listaDB = em.createQuery("SELECT p FROM PkgData p", PkgData.class).getResultList();
for (PkgData p : listaDB)
List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList();
logger.info("Got list of filenames from db");
for (String fileName : listaDB)
{
knownPkg.put(p.fileName, p);
File file = ctx.getFile(p);
File file = ctx.getFile(fileName);
if (!file.exists())
{
logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName()));
em.remove(p);
PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName",
PkgData.class).setParameter("fileName", file.getName()).getSingleResult();
em.remove(pkg);
}
else
{
knownPkg.add(fileName);
}
}
ut.commit();
} catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e)
}
catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e)
{
throw new RuntimeException(e);
}
//Aggiunge sul DB i pacchetti presenti nel filesystem
int cores = Runtime.getRuntime().availableProcessors();
logger.info("Searching for new packages or packages that were modified after being added to the database");
List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
List<File> ls = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
Deque<File> stack = new LinkedList<>(ls);
Thread[] works = new Thread[cores];
for(int i=0; i<cores; i++)
Parser parser = new Parser();
for(File file : pkgfiles)
{
works[i] = mtf.newThread(new SyncWorker(stack));
}
for(Thread t : works)
{
t.start();
}
for(Thread t : works)
{
try
boolean update = !knownPkg.contains(file.getName());
if(!update)
{
t.join();
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getName());
Date result = query.getSingleResult();
if(file.lastModified() > result.getTime())
{
update = true;
}
}
catch (InterruptedException e)
if (update)
{
e.printStackTrace();
try
{
ut.begin();
parseFile(parser, file);
ut.commit();
}
catch (Exception e)
{
logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath()));
try
{
ut.rollback();
}
catch (SystemException e1)
{
throw new RuntimeException(e1);
}
throw new RuntimeException(e);
}
}
}
logger.log(Level.INFO, "Removing obsolete packages");
deleteOld();
logger.info("Repository cleanup completed successfully");
}
private void parseFile(Parser p, File file) throws Exception
@@ -114,7 +143,7 @@ public class PacmanServiceEJB implements PacmanServiceView
PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum);
List<PkgData> savedFiles = hquery.getResultList();
List<PkgData> savedFiles = hquery.getResultList();
if (savedFiles.size() > 0)
{
return;
@@ -124,7 +153,7 @@ public class PacmanServiceEJB implements PacmanServiceView
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", file.getName());
savedFiles = fquery.getResultList();
if(savedFiles.size()>0)
if (savedFiles.size() > 0)
{
em.remove(savedFiles.get(0));
}
@@ -148,7 +177,7 @@ public class PacmanServiceEJB implements PacmanServiceView
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList();
if(savedFiles.size()==0)
if (savedFiles.size() == 0)
{
ut.rollback();
throw new RuntimeException(String.format("Package with name %s not found", filename));
@@ -166,58 +195,9 @@ public class PacmanServiceEJB implements PacmanServiceView
em.remove(pkg);
ut.commit();
logger.log(Level.INFO,String.format("Package %s has been deleted", filename));
logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
}
private class SyncWorker implements Runnable
{
Deque<File> ls;
Parser parser = new Parser();
public SyncWorker(Deque<File> ls)
{
this.ls = ls;
}
@Override
public void run()
{
while(ls.size() > 0)
{
File file;
synchronized (ls)
{
file = ls.pop();
}
try
{
PkgData p = knownPkg.get(file.getName());
if (p == null || file.lastModified() > p.updTimestamp.getTime())
{
ut.begin();
parseFile(parser, file);
ut.commit();
}
}
catch (Exception e)
{
logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath()));
try
{
ut.rollback();
}
catch (SystemException e1)
{
throw new RuntimeException(e1);
}
throw new RuntimeException(e);
}
}
}
}
private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" +
"(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)";
@@ -227,10 +207,10 @@ public class PacmanServiceEJB implements PacmanServiceView
Calendar cutoff = Calendar.getInstance();
cutoff.add(Calendar.YEAR, -2);
query.setParameter("cutoff", cutoff.getTime());
query.setParameter("minVersions", (long)2);
query.setParameter("minVersions", (long) 2);
List<String> list = query.getResultList();
logger.log(Level.INFO, "Removing obsolete packages");
list.forEach((el) -> {
list.forEach((el) ->
{
try
{
deletePackage(el);
@@ -253,25 +233,25 @@ public class PacmanServiceEJB implements PacmanServiceView
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(Long.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate=null, p;
Predicate finalPredicate = null, p;
if(name != null && !name.isEmpty())
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if(version != null && !version.isEmpty())
if (version != null && !version.isEmpty())
{
p=builder.equal(entity.get("version"), version);
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(arch != null && !arch.isEmpty())
if (arch != null && !arch.isEmpty())
{
p=builder.equal(entity.get("arch"), arch);
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(finalPredicate != null)
if (finalPredicate != null)
{
criteriaQuery.select(builder.count(entity)).where(finalPredicate);
}
@@ -284,35 +264,35 @@ public class PacmanServiceEJB implements PacmanServiceView
@Override
@Lock(LockType.READ)
public<T> List<T> listProperty(String property, Map<String,String> equalityConditions, Class<T> cls)
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls)
{
CriteriaBuilder builder = em.getCriteriaBuilder();
CriteriaQuery<T> criteriaQuery = builder.createQuery(cls);
Root<PkgData> entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate=null, p;
Predicate finalPredicate = null, p;
String key = equalityConditions.get("name");
if(key != null && !key.isEmpty())
if (key != null && !key.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), key);
finalPredicate = p;
}
key = equalityConditions.get("version");
if(key != null && !key.isEmpty())
if (key != null && !key.isEmpty())
{
p = builder.equal(entity.get("version"), key);
finalPredicate = p;
}
key = equalityConditions.get("arch");
if(key != null && !key.isEmpty())
if (key != null && !key.isEmpty())
{
p = builder.equal(entity.get("arch"), key);
finalPredicate = p;
}
if(finalPredicate != null)
if (finalPredicate != null)
{
criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate);
}
@@ -335,25 +315,25 @@ public class PacmanServiceEJB implements PacmanServiceView
builder = em.getCriteriaBuilder();
criteriaQuery = builder.createQuery(PkgData.class);
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate=null, p;
Predicate finalPredicate = null, p;
if(name != null && !name.isEmpty())
if (name != null && !name.isEmpty())
{
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if(version != null && !version.isEmpty())
if (version != null && !version.isEmpty())
{
p=builder.equal(entity.get("version"), version);
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(arch != null && !arch.isEmpty())
if (arch != null && !arch.isEmpty())
{
p=builder.equal(entity.get("arch"), arch);
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if(finalPredicate != null)
if (finalPredicate != null)
{
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
@@ -362,11 +342,11 @@ public class PacmanServiceEJB implements PacmanServiceView
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if(pageNumber>=0)
if (pageNumber >= 0)
{
query.setFirstResult(pageNumber*pageSize);
query.setFirstResult(pageNumber * pageSize);
}
if(pageSize>0)
if (pageSize > 0)
{
query.setMaxResults(pageSize);
}

1
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceRemote.java
View File

@@ -9,5 +9,6 @@ import javax.ejb.Remote;
@Remote
public interface PacmanServiceRemote
{
public void syncDB();
public void deletePackage(String filename) throws Exception;
}

73
src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java
View File

@@ -1,11 +1,11 @@
package com.oggio88.jpacrepo.service;
import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.context.ApplicationContext;
import com.oggio88.jpacrepo.context.DefaultConfiguration;
import com.oggio88.jpacrepo.model.StringList;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgList;
import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.model.StringList;
import com.oggio88.jpacrepo.pacbase.Parser;
import com.oggio88.jpacrepo.persistence.QueryEngine;
@@ -21,6 +21,8 @@ import java.io.*;
import java.net.URI;
import java.nio.file.Files;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -71,7 +73,7 @@ public class PacmanWebService
{
return getPackageByFileName(filename);
}
else if (name != null || arch != null || version != null)
else
{
QueryEngine qe = new QueryEngine(PkgData.class);
@@ -82,10 +84,70 @@ public class PacmanWebService
String query = qe.build();
return manageQueryResult(em.createQuery(query).getResultList());
}
else
}
@GET
@Path("searchByName/{name}")
public Response searchPackagebyName(@PathParam("name") String name)
{
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
String query = String.format("SELECT pkgName.id FROM PkgName pkgName WHERE LOWER(pkgName.id) LIKE '%%%s%%' ORDER BY pkgName.id", name);
return Response.ok(em.createQuery(query, String.class).getResultList()).build();
}
@GET
@Path("list/{name}")
public Response getPackage(@PathParam("name") String name)
{
TypedQuery query = em.createQuery("SELECT pkg.version FROM PkgData pkg WHERE pkg.name.id = :name ORDER BY pkg.version", String.class);
query.setParameter("name", name);
return Response.ok(query.getResultList()).build();
}
@GET
@Path("list/{name}/{version}")
public Response getPackage(@PathParam("name") String name, @PathParam("version") String version)
{
TypedQuery query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.name.id = :name AND pkg.version = :version ORDER BY pkg.arch", String.class);
query.setParameter("name", name);
query.setParameter("version", version);
return Response.ok(query.getResultList()).build();
}
@GET
@Path("list/{name}/{version}/{arch}")
public Response getPackage(@PathParam("name") String name, @PathParam("version") String version, @PathParam("arch") String arch)
{
TypedQuery<PkgData> query = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " +
"pkg.name.id = :name AND " +
"pkg.version = :version AND " +
"pkg.arch = :arch " +
"ORDER BY pkg.arch", PkgData.class);
query.setParameter("name", name);
query.setParameter("version", version);
query.setParameter("arch", arch);
return Response.ok(query.getSingleResult()).build();
}
@GET
@Path("map")
public Response getPackageMap()
{
TypedQuery<Object[]> query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class);
SortedMap<String, SortedMap<String, SortedMap<String, String>>> result = new TreeMap<>();
for (Object[] pkg : query.getResultList())
{
throw new NotFoundException();
String name = (String) pkg[0];
String version = (String) pkg[1];
String arch = (String) pkg[2];
String filename = (String) pkg[3];
result.putIfAbsent(name, new TreeMap<>());
SortedMap<String, SortedMap<String, String>> map = result.get(name);
map.putIfAbsent(version, new TreeMap<>());
SortedMap<String, String> map2 = map.get(version);
map2.putIfAbsent(arch, filename);
}
return Response.ok(result).build();
}
@GET
@@ -240,7 +302,6 @@ public class PacmanWebService
private Response manageQueryResult(List<PkgData> list, boolean singleResult)
{
// log.log(Level.INFO, "size: " + list.size());
PkgList pkgList = new PkgList(list);
if (pkgList.size() == 0)
{

16
src/test/java/ClientTest.java
View File

@@ -1,3 +1,4 @@
import com.oggio88.jpacrepo.service.PacmanServiceRemote;
import com.thoughtworks.xstream.XStream;
import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
@@ -133,18 +134,21 @@ public class ClientTest
prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory");
// prop.put(Context.PROVIDER_URL, "http-remoting://localhost:8080");
prop.put(Context.PROVIDER_URL, "http-remoting://odroid-xu4:8080");
prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
// prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
prop.put(Context.SECURITY_PRINCIPAL, "jpacrepo");
prop.put(Context.SECURITY_CREDENTIALS, "password01.");
prop.put(Context.SECURITY_PRINCIPAL, "walter");
prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d");
prop.put("jboss.naming.client.ejb.context", true);
Context context = new InitialContext(prop);
Context ctx = new InitialContext(prop);
traverseJndiNode("/", context);
// final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
final PacmanServiceView service = (PacmanServiceView) ctx.lookup("/jpacrepo/PacmanServiceEJB!org.jpacrepo.service.PacmanService");
List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
System.out.println(new XStream().toXML(pkgs));
final PacmanServiceRemote service = (PacmanServiceRemote) ctx.lookup(
"/jpacrepo-2.0/PacmanServiceEJB!com.oggio88.jpacrepo.service.PacmanServiceRemote"
);
// List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
// System.out.println(new XStream().toXML(pkgs));
service.syncDB();
}
}