woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

temporary commit

Browse Source
This commit is contained in:
Walter Oggioni
2019-05-06 19:59:40 +02:00
committed by Walter Oggioni
parent 8241df995a
commit 1cc066d3f0
2 changed files with 79 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
nim/src/jpacrepo.nim
View File

@@ -25,7 +25,7 @@ type DownloadPanel = ref object
proc newDownloadPanel(parent : Element) : DownloadPanel =
let dp = DownloadPanel()
dp.pkgs = initSet[string]()
dp.pkgs = initHashSet[string]()
htmlTreeAppend(parent):
"div":
classList = ["panel", "panel-default"]
@@ -99,7 +99,7 @@ proc addPkg(dp : DownloadPanel, pkgfile : string) =
dp.pkgs.incl(pkgfile)
let req = newXMLHTTPRequest()
let load_cb = proc(e : Event) =
let sz = parseInt(req.responseText)
let sz = parseInt($req.responseText)
dp.size += sz
dp.updateSize
htmlTreeAppend(dp.listgroup):

214
src/main/java/com/oggio88/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -5,6 +5,7 @@ import com.oggio88.jpacrepo.context.DefaultConfiguration;
import com.oggio88.jpacrepo.model.PkgData;
import com.oggio88.jpacrepo.model.PkgName;
import com.oggio88.jpacrepo.pacbase.Parser;
import lombok.SneakyThrows;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
@@ -27,14 +28,15 @@ import java.util.logging.Logger;
@Startup
@Singleton
@Lock(LockType.READ)
@TransactionManagement(TransactionManagementType.BEAN)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local(PacmanServiceView.class)
@Remote(PacmanServiceRemote.class)
public class PacmanServiceEJB implements PacmanServiceView
{
@PersistenceContext(unitName = "jpacrepo_pu")
private EntityManager em;
public class PacmanServiceEJB implements PacmanServiceView {
@PersistenceUnit(unitName = "jpacrepo_pu")
private EntityManagerFactory emf;
@Inject
@DefaultConfiguration
@@ -48,164 +50,125 @@ public class PacmanServiceEJB implements PacmanServiceView
private final String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
@SneakyThrows
@Asynchronous
@Schedule(hour = "3", minute = "00", persistent = false)
public void syncDB()
{
public void syncDB() {
EntityManager em = emf.createEntityManager();
Set<String> knownPkg;
logger.log(Level.INFO, "Starting repository cleanup");
knownPkg = new HashSet<>();
//Elimina i pacchetti sul DB che non esistono più nel filesystem
logger.info("Searching for packages that are no more in the filesystem");
try
{
ut.setTransactionTimeout(1000);
ut.begin();
List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList();
logger.info("Got list of filenames from db");
for (String fileName : listaDB)
{
File file = ctx.getFile(fileName);
if (!file.exists())
{
logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName()));
PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName",
PkgData.class).setParameter("fileName", file.getName()).getSingleResult();
em.remove(pkg);
}
else
{
knownPkg.add(fileName);
}
ut.setTransactionTimeout(86400);
ut.begin();
List<String> listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", String.class).getResultList();
logger.info("Got list of filenames from db");
for(String fileName : listaDB) {
File file = ctx.getFile(fileName);
if(!file.exists()) {
logger.log(Level.INFO, String.format("Removing package %s which was not found in filesystem", file.getName()));
PkgData pkg = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName",
PkgData.class).setParameter("fileName", file.getName()).getSingleResult();
em.remove(pkg);
} else {
knownPkg.add(fileName);
}
ut.commit();
}
catch (NotSupportedException | SystemException | HeuristicMixedException | RollbackException | HeuristicRollbackException e)
{
throw new RuntimeException(e);
}
logger.info("Searching for new packages or packages that were modified after being added to the database");
List<File> pkgfiles = Arrays.asList(new File(ctx.getSystemProperties().getProperty("RepoFolder")).listFiles((file -> file.getName().endsWith(".pkg.tar.xz"))));
Parser parser = new Parser();
for (File file : pkgfiles)
{
for(File file : pkgfiles) {
boolean update = !knownPkg.contains(file.getName());
if (!update)
{
if(!update) {
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getName());
Date result = query.getSingleResult();
if (file.lastModified() > result.getTime())
{
if(file.lastModified() > result.getTime()) {
update = true;
}
}
if (update)
{
try
{
ut.begin();
parseFile(parser, file);
ut.commit();
}
catch (Exception e)
{
logger.log(Level.INFO, String.format("Error parsing %s", file.getAbsolutePath()));
try
{
ut.rollback();
}
catch (SystemException e1)
{
throw new RuntimeException(e1);
}
throw new RuntimeException(e);
if(update) {
try {
parseFile(em, parser, file);
} catch(Exception e) {
logger.log(Level.SEVERE, String.format("Error parsing %s", file.getAbsolutePath()), e);
}
}
}
logger.log(Level.INFO, "Removing obsolete packages");
deleteOld();
deleteOld(em);
ut.commit();
logger.info("Repository cleanup completed successfully");
ctx.invalidateCache = true;
}
private void parseFile(Parser p, File file) throws Exception
{
private void parseFile(EntityManager em, Parser p, File file) throws Exception {
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum);
try
{
try {
hquery.getSingleResult();
return;
} catch(NoResultException e) {
}
catch (NoResultException e)
{
}
try
{
try {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", file.getName());
PkgData savedFile = fquery.getSingleResult();
em.remove(savedFile);
em.flush();
}
catch (NoResultException e)
{
} catch(NoResultException e) {
}
nquery.setParameter("name", data.name.id);
try
{
try {
PkgName savedName = nquery.getSingleResult();
data.name = savedName;
}
catch (NoResultException e)
{
} catch(NoResultException e) {
}
em.persist(data);
logger.log(Level.INFO, String.format("Persisting package %s", file.getName()));
}
@Override
public void deletePackage(String filename) throws Exception
{
public void deletePackage(String filename) throws Exception {
EntityManager em = emf.createEntityManager();
ut.begin();
deletePackage(em, filename);
ut.commit();
logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
}
public void deletePackage(EntityManager em, String filename) throws Exception {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList();
if (savedFiles.size() == 0)
{
if(savedFiles.size() == 0) {
ut.rollback();
throw new RuntimeException(String.format("Package with name %s not found", filename));
}
PkgData pkg = fquery.getResultList().get(0);
try
{
try {
Files.delete(ctx.getFile(pkg).toPath());
}
catch (IOException e)
{
} catch(IOException e) {
ut.rollback();
throw new RuntimeException(e);
}
em.remove(pkg);
ut.commit();
logger.log(Level.INFO, String.format("Package %s has been deleted", filename));
}
private final String deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.name.id in \n" +
"(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)";
"(SELECT p2.name.id FROM PkgData p2 GROUP BY p2.name.id HAVING count(p2.name.id) > :minVersions\n)";
private void deleteOld()
{
private void deleteOld(EntityManager em) {
TypedQuery<String> query = em.createQuery(deleteQuery, String.class);
Calendar cutoff = Calendar.getInstance();
cutoff.add(Calendar.YEAR, -2);
@@ -214,20 +177,17 @@ public class PacmanServiceEJB implements PacmanServiceView
List<String> list = query.getResultList();
list.forEach((el) ->
{
try
{
try {
deletePackage(el);
}
catch (Exception e)
{
} catch(Exception e) {
throw new RuntimeException(e);
}
});
}
@Override
public long countResults(String name, String version, String arch)
{
public long countResults(String name, String version, String arch) {
EntityManager em = emf.createEntityManager();
CriteriaBuilder builder;
CriteriaQuery<Long> criteriaQuery;
Root<PkgData> entity;
@@ -237,68 +197,56 @@ public class PacmanServiceEJB implements PacmanServiceView
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
if(name != null && !name.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
if(version != null && !version.isEmpty()) {
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
if(arch != null && !arch.isEmpty()) {
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
if(finalPredicate != null) {
criteriaQuery.select(builder.count(entity)).where(finalPredicate);
}
else
{
} else {
criteriaQuery.select(builder.count(entity));
}
return em.createQuery(criteriaQuery).getSingleResult();
}
@Override
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls)
{
public <T> List<T> listProperty(String property, Map<String, String> equalityConditions, Class<T> cls) {
EntityManager em = emf.createEntityManager();
CriteriaBuilder builder = em.getCriteriaBuilder();
CriteriaQuery<T> criteriaQuery = builder.createQuery(cls);
Root<PkgData> entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
String key = equalityConditions.get("name");
if (key != null && !key.isEmpty())
{
if(key != null && !key.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), key);
finalPredicate = p;
}
key = equalityConditions.get("version");
if (key != null && !key.isEmpty())
{
if(key != null && !key.isEmpty()) {
p = builder.equal(entity.get("version"), key);
finalPredicate = p;
}
key = equalityConditions.get("arch");
if (key != null && !key.isEmpty())
{
if(key != null && !key.isEmpty()) {
p = builder.equal(entity.get("arch"), key);
finalPredicate = p;
}
if (finalPredicate != null)
{
if(finalPredicate != null) {
criteriaQuery.select(entity.get(property)).distinct(true).where(finalPredicate);
}
else
{
} else {
criteriaQuery.select(entity.get(property)).distinct(true);
}
return em.createQuery(criteriaQuery).getResultList();
@@ -306,8 +254,8 @@ public class PacmanServiceEJB implements PacmanServiceView
@Override
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize)
{
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize) {
EntityManager em = emf.createEntityManager();
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
@@ -317,37 +265,29 @@ public class PacmanServiceEJB implements PacmanServiceView
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
if(name != null && !name.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
if(version != null && !version.isEmpty()) {
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
if(arch != null && !arch.isEmpty()) {
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
if(finalPredicate != null) {
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
else
{
} else {
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0)
{
if(pageNumber >= 0) {
query.setFirstResult(pageNumber * pageSize);
}
if (pageSize > 0)
{
if(pageSize > 0) {
query.setMaxResults(pageSize);
}
return query.getResultList();