woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

temporary commit

Browse Source
This commit is contained in:
Walter Oggioni
2019-05-06 19:59:40 +02:00
committed by Walter Oggioni
parent 8241df995a
commit 1cc066d3f0
2 changed files with 79 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
nim/src/jpacrepo.nim
View File

@@ -25,7 +25,7 @@ type DownloadPanel = ref object
proc newDownloadPanel(parent : Element) : DownloadPanel = proc newDownloadPanel(parent : Element) : DownloadPanel =
let dp = DownloadPanel() let dp = DownloadPanel()
dp.pkgs = initSet[string]() dp.pkgs = initHashSet[string]()
htmlTreeAppend(parent): htmlTreeAppend(parent):
"div": "div":
classList = ["panel", "panel-default"] classList = ["panel", "panel-default"]
@@ -99,7 +99,7 @@ proc addPkg(dp : DownloadPanel, pkgfile : string) =
dp.pkgs.incl(pkgfile) dp.pkgs.incl(pkgfile)
let req = newXMLHTTPRequest() let req = newXMLHTTPRequest()
let load_cb = proc(e : Event) = let load_cb = proc(e : Event) =
let sz = parseInt(req.responseText) let sz = parseInt($req.responseText)
dp.size += sz dp.size += sz
dp.updateSize dp.updateSize
htmlTreeAppend(dp.listgroup): htmlTreeAppend(dp.listgroup):

194
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -5,6 +5,7 @@ import com.oggio88.jpacrepo.context.DefaultConfiguration;
import com.oggio88.jpacrepo.model.PkgData; import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgName; import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.pacbase.Parser; import com.oggio88.jpacrepo.pacbase.Parser;
import lombok.SneakyThrows;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.Resource; import javax.annotation.Resource;
@@ -27,14 +28,15 @@ import java.util.logging.Logger;
@Startup @Startup
@Singleton @Singleton
@Lock(LockType.READ)
@TransactionManagement(TransactionManagementType.BEAN) @TransactionManagement(TransactionManagementType.BEAN)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER) @ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local(PacmanServiceView.class) @Local(PacmanServiceView.class)
@Remote(PacmanServiceRemote.class) @Remote(PacmanServiceRemote.class)
public class PacmanServiceEJB implements PacmanServiceView public class PacmanServiceEJB implements PacmanServiceView {
{
@PersistenceContext(unitName = "jpacrepo_pu") @PersistenceUnit(unitName = "jpacrepo_pu")
private EntityManager em; private EntityManagerFactory emf;
@Inject @Inject
@DefaultConfiguration @DefaultConfiguration
@@ -48,164 +50,125 @@ public class PacmanServiceEJB implements PacmanServiceView
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name"; private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"; private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
@SneakyThrows
@Asynchronous @Asynchronous
@Schedule(hour = "3", minute = "00", persistent = false) @Schedule(hour = "3", minute = "00", persistent = false)
public void syncDB() public void syncDB() {
{ EntityManager em = emf.createEntityManager();
Set<String> knownPkg; Set<String> knownPkg;
logger.log(Level.INFO, "Starting repository cleanup"); logger.log(Level.INFO, "Starting repository cleanup");
knownPkg = new HashSet<>(); knownPkg = new HashSet<>();
//Elimina i pacchetti sul DB che non esistono più nel filesystem //Elimina i pacchetti sul DB che non esistono più nel filesystem
logger.info("Searching for packages that are no more in the filesystem"); logger.info("Searching for packages that are no more in the filesystem");
try ut.setTransactionTimeout(86400);
{
ut.setTransactionTimeout(1000);
ut.begin(); ut.begin();
List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList(); List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList();
logger.info("Got list of filenames from db"); logger.info("Got list of filenames from db");
for (String fileName : listaDB) for(String fileName : listaDB) {
{
File file = ctx.getFile(fileName); File file = ctx.getFile(fileName);
if (!file.exists()) if(!file.exists()) {
{
logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName())); logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName()));
PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName",
PkgData.class).setParameter("fileName", file.getName()).getSingleResult(); PkgData.class).setParameter("fileName", file.getName()).getSingleResult();
em.remove(pkg); em.remove(pkg);
} } else {
else
{
knownPkg.add(fileName); knownPkg.add(fileName);
} }
} }
ut.commit();
}
catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e)
{
throw new RuntimeException(e);
}
logger.info("Searching for new packages or packages that were modified after being added to the database"); logger.info("Searching for new packages or packages that were modified after being added to the database");
List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz")))); List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
Parser parser = new Parser(); Parser parser = new Parser();
for (File file : pkgfiles) for(File file : pkgfiles) {
{
boolean update = !knownPkg.contains(file.getName()); boolean update = !knownPkg.contains(file.getName());
if (!update) if(!update) {
{
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class); TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getName()); query.setParameter("filename", file.getName());
Date result = query.getSingleResult(); Date result = query.getSingleResult();
if (file.lastModified() > result.getTime()) if(file.lastModified() > result.getTime()) {
{
update = true; update = true;
} }
} }
if (update) if(update) {
{ try {
try parseFile(em, parser, file);
{ } catch(Exception e) {
ut.begin(); logger.log(Level.SEVERE, String.format("Error parsing %s", file.getAbsolutePath()), e);
parseFile(parser, file);
ut.commit();
}
catch (Exception e)
{
logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath()));
try
{
ut.rollback();
}
catch (SystemException e1)
{
throw new RuntimeException(e1);
}
throw new RuntimeException(e);
} }
} }
} }
logger.log(Level.INFO, "Removing obsolete packages"); logger.log(Level.INFO, "Removing obsolete packages");
deleteOld(); deleteOld(em);
ut.commit();
logger.info("Repository cleanup completed successfully"); logger.info("Repository cleanup completed successfully");
ctx.invalidateCache = true; ctx.invalidateCache = true;
} }
private void parseFile(Parser p, File file) throws Exception private void parseFile(EntityManager em, Parser p, File file) throws Exception {
{
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class); TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class); TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
PkgData data = p.parseFile(file); PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum); hquery.setParameter("md5sum", data.md5sum);
try try {
{
hquery.getSingleResult(); hquery.getSingleResult();
return; return;
} catch(NoResultException e) {
} }
catch (NoResultException e) try {
{
}
try
{
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class); TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", file.getName()); fquery.setParameter("fileName", file.getName());
PkgData savedFile = fquery.getSingleResult(); PkgData savedFile = fquery.getSingleResult();
em.remove(savedFile); em.remove(savedFile);
em.flush(); em.flush();
} } catch(NoResultException e) {
catch (NoResultException e)
{
} }
nquery.setParameter("name", data.name.id); nquery.setParameter("name", data.name.id);
try try {
{
PkgName savedName = nquery.getSingleResult(); PkgName savedName = nquery.getSingleResult();
data.name = savedName; data.name = savedName;
} } catch(NoResultException e) {
catch (NoResultException e)
{
} }
em.persist(data); em.persist(data);
logger.log(Level.INFO, String.format("Persisting package %s", file.getName())); logger.log(Level.INFO, String.format("Persisting package %s", file.getName()));
} }
@Override @Override
public void deletePackage(String filename) throws Exception public void deletePackage(String filename) throws Exception {
{ EntityManager em = emf.createEntityManager();
ut.begin(); ut.begin();
deletePackage(em, filename);
ut.commit();
logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
}
public void deletePackage(EntityManager em, String filename) throws Exception {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class); TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", filename); fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList(); List<PkgData> savedFiles = fquery.getResultList();
if (savedFiles.size() == 0) if(savedFiles.size() == 0) {
{
ut.rollback(); ut.rollback();
throw new RuntimeException(String.format("Package with name %s not found", filename)); throw new RuntimeException(String.format("Package with name %s not found", filename));
} }
PkgData pkg = fquery.getResultList().get(0); PkgData pkg = fquery.getResultList().get(0);
try try {
{
Files.delete(ctx.getFile(pkg).toPath()); Files.delete(ctx.getFile(pkg).toPath());
} } catch(IOException e) {
catch (IOException e)
{
ut.rollback(); ut.rollback();
throw new RuntimeException(e); throw new RuntimeException(e);
} }
em.remove(pkg); em.remove(pkg);
ut.commit();
logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
} }
private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" + private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" +
"(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)"; "(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)";
private void deleteOld() private void deleteOld(EntityManager em) {
{
TypedQuery<String> query = em.createQuery(deleteQuery, String.class); TypedQuery<String> query = em.createQuery(deleteQuery, String.class);
Calendar cutoff = Calendar.getInstance(); Calendar cutoff = Calendar.getInstance();
cutoff.add(Calendar.YEAR, -2); cutoff.add(Calendar.YEAR, -2);
@@ -214,20 +177,17 @@ public class PacmanServiceEJB implements PacmanServiceView
List<String> list = query.getResultList(); List<String> list = query.getResultList();
list.forEach((el) -> list.forEach((el) ->
{ {
try try {
{
deletePackage(el); deletePackage(el);
} } catch(Exception e) {
catch (Exception e)
{
throw new RuntimeException(e); throw new RuntimeException(e);
} }
}); });
} }
@Override @Override
public long countResults(String name, String version, String arch) public long countResults(String name, String version, String arch) {
{ EntityManager em = emf.createEntityManager();
CriteriaBuilder builder; CriteriaBuilder builder;
CriteriaQuery<Long> criteriaQuery; CriteriaQuery<Long> criteriaQuery;
Root<PkgData> entity; Root<PkgData> entity;
@@ -237,68 +197,56 @@ public class PacmanServiceEJB implements PacmanServiceView
entity = criteriaQuery.from(PkgData.class); entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p; Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty()) if(name != null && !name.isEmpty()) {
{
p = builder.equal(entity.get("name").get("id"), name); p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p; finalPredicate = p;
} }
if (version != null && !version.isEmpty()) if(version != null && !version.isEmpty()) {
{
p = builder.equal(entity.get("version"), version); p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
} }
if (arch != null && !arch.isEmpty()) if(arch != null && !arch.isEmpty()) {
{
p = builder.equal(entity.get("arch"), arch); p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
} }
if (finalPredicate != null) if(finalPredicate != null) {
{
criteriaQuery.select(builder.count(entity)).where(finalPredicate); criteriaQuery.select(builder.count(entity)).where(finalPredicate);
} } else {
else
{
criteriaQuery.select(builder.count(entity)); criteriaQuery.select(builder.count(entity));
} }
return em.createQuery(criteriaQuery).getSingleResult(); return em.createQuery(criteriaQuery).getSingleResult();
} }
@Override @Override
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls) public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls) {
{ EntityManager em = emf.createEntityManager();
CriteriaBuilder builder = em.getCriteriaBuilder(); CriteriaBuilder builder = em.getCriteriaBuilder();
CriteriaQuery<T> criteriaQuery = builder.createQuery(cls); CriteriaQuery<T> criteriaQuery = builder.createQuery(cls);
Root<PkgData> entity = criteriaQuery.from(PkgData.class); Root<PkgData> entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p; Predicate finalPredicate = null, p;
String key = equalityConditions.get("name"); String key = equalityConditions.get("name");
if (key != null && !key.isEmpty()) if(key != null && !key.isEmpty()) {
{
p = builder.equal(entity.get("name").get("id"), key); p = builder.equal(entity.get("name").get("id"), key);
finalPredicate = p; finalPredicate = p;
} }
key = equalityConditions.get("version"); key = equalityConditions.get("version");
if (key != null && !key.isEmpty()) if(key != null && !key.isEmpty()) {
{
p = builder.equal(entity.get("version"), key); p = builder.equal(entity.get("version"), key);
finalPredicate = p; finalPredicate = p;
} }
key = equalityConditions.get("arch"); key = equalityConditions.get("arch");
if (key != null && !key.isEmpty()) if(key != null && !key.isEmpty()) {
{
p = builder.equal(entity.get("arch"), key); p = builder.equal(entity.get("arch"), key);
finalPredicate = p; finalPredicate = p;
} }
if (finalPredicate != null) if(finalPredicate != null) {
{
criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate); criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate);
} } else {
else
{
criteriaQuery.select(entity.get(property)).distinct(true); criteriaQuery.select(entity.get(property)).distinct(true);
} }
return em.createQuery(criteriaQuery).getResultList(); return em.createQuery(criteriaQuery).getResultList();
@@ -306,8 +254,8 @@ public class PacmanServiceEJB implements PacmanServiceView
@Override @Override
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize) public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize) {
{ EntityManager em = emf.createEntityManager();
CriteriaBuilder builder; CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery; CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity; Root<PkgData> entity;
@@ -317,37 +265,29 @@ public class PacmanServiceEJB implements PacmanServiceView
entity = criteriaQuery.from(PkgData.class); entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p; Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty()) if(name != null && !name.isEmpty()) {
{
p = builder.equal(entity.get("name").get("id"), name); p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p; finalPredicate = p;
} }
if (version != null && !version.isEmpty()) if(version != null && !version.isEmpty()) {
{
p = builder.equal(entity.get("version"), version); p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
} }
if (arch != null && !arch.isEmpty()) if(arch != null && !arch.isEmpty()) {
{
p = builder.equal(entity.get("arch"), arch); p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p; finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
} }
if (finalPredicate != null) if(finalPredicate != null) {
{
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName"))); criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
} } else {
else
{
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName"))); criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
} }
TypedQuery<PkgData> query = em.createQuery(criteriaQuery); TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0) if(pageNumber >= 0) {
{
query.setFirstResult(pageNumber * pageSize); query.setFirstResult(pageNumber * pageSize);
} }
if (pageSize > 0) if(pageSize > 0) {
{
query.setMaxResults(pageSize); query.setMaxResults(pageSize);
} }
return query.getResultList(); return query.getResultList();