woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

java rewrite

Browse Source
This commit is contained in:
Walter Oggioni
2022-06-03 23:59:59 +08:00
parent 76608d5713
commit b59dcfd93c
68 changed files with 2218 additions and 1769 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/net/woggioni/jpacrepo/config/AppConfig.java Normal file
View File

@@ -0,0 +1,62 @@
package net.woggioni.jpacrepo.config;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.api.model.PkgData;
import net.woggioni.jpacrepo.persistence.InitialSchemaAction;
import net.woggioni.jwo.BiFun;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
@RequiredArgsConstructor(access = AccessLevel.PUBLIC)
public class AppConfig {
@Getter
private final Path repoFolder;
@Getter
private final InitialSchemaAction initialSchemaAction;
@Getter
private final String dataSourceJndi;
@Getter
private final AtomicBoolean invalidateCache = new AtomicBoolean(true);
public Path getFile(PkgData pkg) {
return repoFolder.resolve(pkg.getFileName());
}
public Path getFile(String fileName) {
return repoFolder.resolve(fileName);
}
@SneakyThrows
public static AppConfig newInstance(String propertyFile) {
Properties properties = new Properties();
var path = Path.of(propertyFile);
if (Files.exists(path)) {
try(InputStream is = Files.newInputStream(path)) {
properties.load(is);
}
}
BiFun<String, String, String> getProperty = (String key, String defaultValue) -> {
key = "net.woggioni.jpacrepo." + key;
String result = System.getProperty(key);
if(result == null) {
result = properties.getProperty(key, defaultValue);
}
return result;
};
return new AppConfig(
Path.of(getProperty.apply("RepoFolder", "/var/cache/pacman/pkg")),
InitialSchemaAction.from(getProperty.apply("InitialSchemaAction", "none")),
getProperty.apply("dataSourceJndi", "java:comp/DefaultDataSource"));
}
}

23
src/main/java/net/woggioni/jpacrepo/factory/BeanFactory.java Normal file
View File

@@ -0,0 +1,23 @@
package net.woggioni.jpacrepo.factory;
import net.woggioni.jpacrepo.config.AppConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.enterprise.inject.Produces;
import javax.enterprise.inject.spi.InjectionPoint;
public class BeanFactory {
@Produces
public AppConfig newAppConfig() {
return AppConfig.newInstance(
System.getProperty("net.woggioni.jpacrepo.configuration.file",
"/etc/jpacrepo/server.properties"));
}
@Produces
public Logger createLogger(InjectionPoint injectionPoint) {
return LoggerFactory.getLogger(injectionPoint.getMember().getDeclaringClass().getName());
}
}

22
src/main/java/net/woggioni/jpacrepo/factory/PersistenceUnitFactory.java Normal file
View File

@@ -0,0 +1,22 @@
package net.woggioni.jpacrepo.factory;
import net.woggioni.jpacrepo.config.AppConfig;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.inject.Produces;
import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence;
import java.util.Properties;
public class PersistenceUnitFactory {
@Produces
@ApplicationScoped
private EntityManagerFactory createEntityManagerFactory(AppConfig appConfig) {
Properties properties = new Properties();
properties.put("javax.persistence.schema-generation.database.action",
appConfig.getInitialSchemaAction().getValue());
properties.put("javax.persistence.jtaDataSource", appConfig.getDataSourceJndi());
return Persistence.createEntityManagerFactory("jpacrepo_pu", properties);
}
}

45
src/main/java/net/woggioni/jpacrepo/persistence/InitialSchemaAction.java Normal file
View File

@@ -0,0 +1,45 @@
package net.woggioni.jpacrepo.persistence;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import net.woggioni.jwo.JWO;
import java.util.Collections;
import java.util.Map;
import java.util.TreeMap;
@RequiredArgsConstructor
public enum InitialSchemaAction {
NONE("none"), CREATE("create"), DROP("drop"), DROP_AND_CREATE("drop-and-create");
@Getter
private final String value;
@Override
public String toString() {
return value;
}
private static final Map<String, InitialSchemaAction> valueMap;
static {
Map<String, InitialSchemaAction> result = new TreeMap<>();
for(InitialSchemaAction initialSchemaAction : values()) {
result.put(initialSchemaAction.value, initialSchemaAction);
}
valueMap = Collections.unmodifiableMap(result);
}
@SneakyThrows
public static InitialSchemaAction from(String s) {
InitialSchemaAction result = valueMap.get(s);
if(result == null) {
throw JWO.newThrowable(IllegalArgumentException.class,
"Unknown initial schema action '%s'", s);
}
return result;
}
}

81
src/main/java/net/woggioni/jpacrepo/persistence/QueryEngine.java
View File

@@ -1,6 +1,7 @@
package net.woggioni.jpacrepo.persistence;
import net.woggioni.jpacrepo.pacbase.PkgData;
import net.woggioni.jpacrepo.api.model.PkgData;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
@@ -11,66 +12,52 @@ import javax.persistence.criteria.Root;
import java.util.ArrayList;
import java.util.List;
/**
* Created by walter on 29/03/15.
*/
public class QueryEngine
{
private String entityName;
private String query;
private List<String> where;
public QueryEngine(Class<?> cls)
{
public QueryEngine(Class<?> cls) {
query = String.format("SELECT e FROM %s e", cls.getSimpleName());
this.entityName = cls.getSimpleName();
where = new ArrayList<>();
}
public QueryEngine(String entityName)
{
public QueryEngine(String entityName) {
query = String.format("SELECT e FROM %s e", entityName);
where = new ArrayList<>();
}
public QueryEngine select(String... fields)
{
public QueryEngine select(String... fields) {
String[] strarr = new String[fields.length];
for (int i = 0; i < fields.length; i++)
{
for (int i = 0; i < fields.length; i++) {
strarr[i] = "e." + fields[i];
}
query = "SELECT " + String.join(",", strarr) + " FROM " + entityName + " e";
return this;
}
public QueryEngine select()
{
public QueryEngine select() {
query = String.format("SELECT e FROM %s e", entityName);
return this;
}
public QueryEngine where(String field, String operator, String value)
{
public QueryEngine where(String field, String operator, String value) {
where.add(String.format("e.%s %s '%s'", field, operator, value));
return this;
}
public String build()
{
if (where.isEmpty())
{
public String build() {
if (where.isEmpty()) {
return query;
}
else
{
} else {
return query + " WHERE " + String.join(" AND ", where);
}
}
public static List<PkgData> searchPackage(
EntityManager em, String name, String version, String arch, int pageNumber, int pageSize, String fileName)
{
EntityManager em, String name, String version, String arch, int pageNumber, int pageSize, String fileName) {
CriteriaBuilder builder;
CriteriaQuery<PkgData> criteriaQuery;
Root<PkgData> entity;
@@ -80,49 +67,39 @@ public class QueryEngine
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
if (name != null && !name.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
if (version != null && !version.isEmpty()) {
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
if (arch != null && !arch.isEmpty()) {
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (fileName != null && !fileName.isEmpty())
{
if (fileName != null && !fileName.isEmpty()) {
p = builder.equal(entity.get("fileName"), fileName);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
if (finalPredicate != null) {
criteriaQuery.select(entity).where(finalPredicate).orderBy(builder.asc(entity.get("fileName")));
}
else
{
} else {
criteriaQuery.select(entity).orderBy(builder.asc(entity.get("fileName")));
}
TypedQuery<PkgData> query = em.createQuery(criteriaQuery);
if (pageNumber >= 0)
{
if (pageNumber >= 0) {
query.setFirstResult(pageNumber * pageSize);
}
if (pageSize > 0)
{
if (pageSize > 0) {
query.setMaxResults(pageSize);
}
return query.getResultList();
}
public static long countResults(EntityManager em, String name, String version, String arch)
{
public static long countResults(EntityManager em, String name, String version, String arch) {
CriteriaBuilder builder;
CriteriaQuery<Long> criteriaQuery;
Root<PkgData> entity;
@@ -132,28 +109,22 @@ public class QueryEngine
entity = criteriaQuery.from(PkgData.class);
Predicate finalPredicate = null, p;
if (name != null && !name.isEmpty())
{
if (name != null && !name.isEmpty()) {
p = builder.equal(entity.get("name").get("id"), name);
finalPredicate = p;
}
if (version != null && !version.isEmpty())
{
if (version != null && !version.isEmpty()) {
p = builder.equal(entity.get("version"), version);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (arch != null && !arch.isEmpty())
{
if (arch != null && !arch.isEmpty()) {
p = builder.equal(entity.get("arch"), arch);
finalPredicate = finalPredicate != null ? builder.and(finalPredicate, p) : p;
}
if (finalPredicate != null)
{
if (finalPredicate != null) {
criteriaQuery.select(builder.count(entity)).where(finalPredicate);
}
else
{
} else {
criteriaQuery.select(builder.count(entity));
}
return em.createQuery(criteriaQuery).getSingleResult();

213
src/main/java/net/woggioni/jpacrepo/service/PacmanServiceEJB.java Normal file
View File

@@ -0,0 +1,213 @@
package net.woggioni.jpacrepo.service;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.api.model.PkgData;
import net.woggioni.jpacrepo.api.service.PacmanServiceLocal;
import net.woggioni.jpacrepo.api.service.PacmanServiceRemote;
import net.woggioni.jpacrepo.config.AppConfig;
import net.woggioni.jpacrepo.impl.model.CompressionFormatImpl;
import net.woggioni.jpacrepo.impl.model.PkgDataImpl;
import net.woggioni.jpacrepo.persistence.QueryEngine;
import net.woggioni.jwo.Con;
import net.woggioni.jwo.JWO;
import net.woggioni.jwo.Sup;
import org.slf4j.Logger;
import javax.annotation.Resource;
import javax.ejb.Asynchronous;
import javax.ejb.ConcurrencyManagement;
import javax.ejb.ConcurrencyManagementType;
import javax.ejb.Local;
import javax.ejb.Lock;
import javax.ejb.LockType;
import javax.ejb.Remote;
import javax.ejb.Schedule;
import javax.ejb.Singleton;
import javax.ejb.Startup;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.ejb.TransactionManagement;
import javax.ejb.TransactionManagementType;
import javax.enterprise.concurrent.ManagedExecutorService;
import javax.inject.Inject;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.TypedQuery;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Startup
@Singleton
@Lock(LockType.READ)
@TransactionManagement(TransactionManagementType.CONTAINER)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local({PacmanServiceLocal.class})
@Remote({PacmanServiceRemote.class})
public class PacmanServiceEJB implements PacmanServiceLocal {
@Inject
private EntityManagerFactory emf;
@Inject
private AppConfig ctx;
@Inject
private Logger logger;
@Resource(name = "DefaultManagedExecutorService")
private ManagedExecutorService executor;
private final static String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name";
private final static String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum";
private final static String hashQueryCount = "SELECT count(pdata) FROM PkgData pdata WHERE md5sum = :md5sum";
private void deletePkgData(EntityManager em, PkgData pkgData) {
em.remove(pkgData);
}
@Override
@SneakyThrows
@Asynchronous
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Schedule(hour = "4", minute = "00", persistent = false)
public void syncDB() {
EntityManager em = emf.createEntityManager();
try {
logger.info("Starting repository cleanup");
//Removes from DB the packages that have been deleted from filesystem
logger.info("Searching for packages that are no more in the filesystem");
List<String> resultList = em.createQuery("SELECT p.fileName FROM PkgData p", String.class)
.getResultList();
logger.info("Got list of filenames from db");
Set<String> knownPkg = resultList.stream().filter(fileName -> {
Path file = ctx.getFile(fileName);
boolean result = Files.exists(file);
if (!result) {
logger.info("Removing package {} which was not found in filesystem", file.getFileName());
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", PkgData.class)
.setParameter("fileName", file.getFileName().toString())
.getResultList().forEach(pkgData -> deletePkgData(em, pkgData));
}
return result;
}).collect(Collectors.toUnmodifiableSet());
logger.info("Searching for new packages or packages that were modified after being added to the database");
CompletionService<PkgData> completionService = new ExecutorCompletionService<>(executor);
final Set<Future<PkgData>> inProgress = new HashSet<>();
final int maxInProgress = Runtime.getRuntime().availableProcessors() * 5;
Con<Boolean> persistPackages = (Boolean drain) -> {
while ((drain && inProgress.size() > 0) || inProgress.size() > maxInProgress) {
Future<PkgData> future = completionService.poll(1, TimeUnit.SECONDS);
inProgress.remove(future);
PkgData pkgData;
try {
pkgData = future.get();
} catch (ExecutionException ee) {
throw ee.getCause();
}
persistPackage(em, pkgData);
}
};
Files.list(ctx.getRepoFolder()).filter((Path file) -> {
String name = file.getFileName().toString();
return name.endsWith(".pkg.tar.xz") || name.endsWith(".pkg.tar.zst") || name.endsWith(".pkg.tar.gz");
}).forEach((Con<Path>) file -> {
if (!knownPkg.contains(file.getFileName().toString()) || ((Sup<Boolean>) () -> {
TypedQuery<Date> query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", Date.class);
query.setParameter("filename", file.getFileName().toString());
Date result = query.getSingleResult();
return Files.getLastModifiedTime(file).toMillis() > result.getTime();
}).get()) {
inProgress.add(completionService.submit(() -> {
try {
return PkgDataImpl.parseFile(file, CompressionFormatImpl.guess(file));
} catch (Exception ex) {
logger.error(String.format("Error parsing '%s'", file.toAbsolutePath()), ex);
throw ex;
}
}));
}
persistPackages.accept(false);
});
persistPackages.accept(true);
logger.info("Removing obsolete packages");
deleteOld(em);
logger.info("Repository cleanup completed successfully");
ctx.getInvalidateCache().set(true);
} finally {
em.close();
}
}
private void persistPackage(EntityManager em, PkgData pkgData) {
TypedQuery<Long> hquery = em.createQuery(hashQueryCount, Long.class);
hquery.setParameter("md5sum", pkgData.getMd5sum());
if (hquery.getSingleResult() == 0) {
TypedQuery<PkgData> fquery =
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", pkgData.getFileName());
fquery.getResultList().forEach(p -> deletePkgData(em, p));
em.persist(pkgData);
logger.info("Persisting package {}", pkgData.getFileName());
}
}
@Override
@TransactionAttribute(TransactionAttributeType.REQUIRED)
public void deletePackage(String filename) {
EntityManager em = emf.createEntityManager();
deletePackage(em, filename);
logger.info("Package {} has been deleted", filename);
}
@SneakyThrows
private void deletePackage(EntityManager em, String filename) {
TypedQuery<PkgData> fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", PkgData.class);
fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList();
if (savedFiles.size() == 0) {
throw JWO.newThrowable(IllegalArgumentException.class, "Package with name %s not found", filename);
}
PkgData pkg = fquery.getResultList().get(0);
Files.delete(ctx.getFile(pkg));
em.remove(pkg);
}
static private final String deleteQuery =
"SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.id.name in \n" + "(SELECT p2.id.name FROM PkgData p2 GROUP BY p2.id.name HAVING count(p2.id.name) > :minVersions\n)";
private void deleteOld(EntityManager em) {
TypedQuery<String> query = em.createQuery(deleteQuery, String.class);
Calendar cutoff = Calendar.getInstance();
cutoff.add(Calendar.YEAR, -2);
query.setParameter("cutoff", OffsetDateTime.now());
query.setParameter("minVersions", 2L);
List<String> list = query.getResultList();
list.forEach(this::deletePackage);
}
@Override
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
public long countResults(String name, String version, String arch) {
EntityManager em = emf.createEntityManager();
return QueryEngine.countResults(em, name, version, arch);
}
@Override
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
public List<PkgData> searchPackage(String name, String version, String arch, int pageNumber, int pageSize, String fileName) {
EntityManager em = emf.createEntityManager();
return QueryEngine.searchPackage(em, name, version, arch, pageNumber, pageSize, null);
}
}

427
src/main/java/net/woggioni/jpacrepo/service/PacmanWebService.java Normal file
View File

@@ -0,0 +1,427 @@
package net.woggioni.jpacrepo.service;
import lombok.SneakyThrows;
import lombok.val;
import net.woggioni.jpacrepo.api.model.PkgData;
import net.woggioni.jpacrepo.api.model.PkgId;
import net.woggioni.jpacrepo.api.service.PacmanServiceLocal;
import net.woggioni.jpacrepo.config.AppConfig;
import net.woggioni.jpacrepo.impl.model.CompressionFormatImpl;
import net.woggioni.jpacrepo.impl.model.PkgDataImpl;
import net.woggioni.jpacrepo.service.wire.PkgDataList;
import net.woggioni.jpacrepo.service.wire.PkgTuple;
import net.woggioni.jpacrepo.service.wire.StringList;
import net.woggioni.jpacrepo.version.PkgIdComparator;
import net.woggioni.jpacrepo.version.VersionComparator;
import net.woggioni.jwo.CollectionUtils;
import net.woggioni.jwo.Con;
import net.woggioni.jwo.JWO;
import net.woggioni.jwo.Tuple2;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.slf4j.Logger;
import javax.ejb.ConcurrencyManagement;
import javax.ejb.ConcurrencyManagementType;
import javax.ejb.Singleton;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.ejb.TransactionManagement;
import javax.ejb.TransactionManagementType;
import javax.inject.Inject;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.NoResultException;
import javax.persistence.NonUniqueResultException;
import javax.persistence.TypedQuery;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.Consumes;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.MatrixParam;
import javax.ws.rs.NotFoundException;
import javax.ws.rs.OPTIONS;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.CacheControl;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.EntityTag;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Request;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
@Singleton
@Path("/pkg")
@ConcurrencyManagement(ConcurrencyManagementType.BEAN)
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@TransactionManagement(TransactionManagementType.CONTAINER)
public class PacmanWebService {
private NavigableMap<PkgId, PkgTuple> cachedMap;
@Inject
private EntityManagerFactory emf;
@Inject
private Logger log;
@Inject
private PacmanServiceLocal service;
@Inject
private AppConfig ctx;
private NavigableMap<PkgId, PkgTuple> getCachedMap() {
NavigableMap<PkgId, PkgTuple> result = null;
if (!ctx.getInvalidateCache().get()) {
result = cachedMap;
}
if (result == null) {
synchronized(this) {
result = cachedMap;
if (result == null) {
EntityManager em = emf.createEntityManager();
TypedQuery<Object[]> query = em.createQuery(
"SELECT pkg.id.name, pkg.id.version, pkg.id.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.id.name, pkg.id.version, pkg.id.arch",
Object[].class);
cachedMap = query.getResultStream()
.map((Object[] pkg) -> {
String name = (String) pkg[0];
String version = (String) pkg[1];
String arch = (String) pkg[2];
String filename = (String) pkg[3];
long size = (long) pkg[4];
String md5sum = (String) pkg[5];
PkgTuple tuple = new PkgTuple();
tuple.setFilename(filename);
tuple.setSize(size);
tuple.setMd5sum(md5sum);
PkgId id = new PkgId();
id.setName(name);
id.setVersion(version);
id.setArch(arch);
return Tuple2.newInstance(id, tuple);
}).collect(
CollectionUtils.toUnmodifiableTreeMap(
Tuple2<PkgId, PkgTuple>::get_1,
Tuple2<PkgId, PkgTuple>::get_2,
PkgIdComparator.getComparator())
);
ctx.getInvalidateCache().set(false);
result = cachedMap;
}
}
}
return result;
}
private Response manageQueryResult(List<PkgData> list) {
return manageQueryResult(list, false);
}
private Response manageQueryResult(List<PkgData> list, boolean singleResult) {
if (list.isEmpty()) throw new NotFoundException();
else if (singleResult) {
if (list.size() == 1) return Response.ok(list.get(0)).build();
else throw new NonUniqueResultException("The returned list does not contain a single element");
} else {
return Response.ok(new PkgDataList(list)).build();
}
}
@GET
@Path("searchByName/{name}")
public Response searchByName(@PathParam("name") String name) {
val em = emf.createEntityManager();
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST);
String query = String.format("SELECT pkgId.name FROM PkgId pkgId WHERE LOWER(pkgId.name) LIKE '%%%s%%' ORDER BY pkgId.name", name);
return Response.ok(em.createQuery(query, String.class).getResultList()).build();
}
@GET
@Path("searchByHash/{md5sum}")
public Response searchByHash(@PathParam("md5sum") String md5sum) {
return getPackageByHash(md5sum);
}
@GET
@Path("list/{name}")
public Response getPackage(@PathParam("name") String name) {
EntityManager em = emf.createEntityManager();
TypedQuery<String> query = em.createQuery("SELECT pkg.id.version FROM PkgData pkg WHERE pkg.id.name = :name ORDER BY pkg.id.version", String.class);
query.setParameter("name", name);
return Response.ok(new StringList(query.getResultList())).build();
}
@GET
@Path("list/{name}/{version}")
public Response getPackage(@PathParam("name") String name, @PathParam("version") String version) {
EntityManager em = emf.createEntityManager();
TypedQuery<String> query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.id.name = :name AND pkg.id.version = :version ORDER BY pkg.id.arch", String.class);
query.setParameter("name", name);
query.setParameter("version", version);
return Response.ok(new StringList(query.getResultList())).build();
}
@GET
@Path("list/{name}/{version}/{arch}")
public Response getPackage(@PathParam("name") String name, @PathParam("version") String version, @PathParam("arch") String arch) {
EntityManager em = emf.createEntityManager();
TypedQuery<PkgData> query = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.id.name = :name AND " + "pkg.id.version = :version AND " + "pkg.id.arch = :arch " + "ORDER BY pkg.arch", PkgData.class);
query.setParameter("name", name);
query.setParameter("version", version);
query.setParameter("arch", arch);
return Response.ok(query.getSingleResult()).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("map")
public Response getPackageMap(@Context Request request) {
CacheControl cc = new CacheControl();
cc.setMaxAge(86400);
cc.setMustRevalidate(true);
cc.setNoCache(true);
NavigableMap<PkgId, PkgTuple> cachedMap = getCachedMap();
EntityTag etag = new EntityTag(Integer.toString(cachedMap.hashCode()), false);
Response.ResponseBuilder builder = request.evaluatePreconditions(etag);
if (builder == null) {
TreeMap<String, TreeMap<String, Map<String, NavigableSet<PkgTuple>>>> result = cachedMap.entrySet().stream().collect(
Collectors.groupingBy((Map.Entry<PkgId, PkgTuple> entry) -> entry.getKey().getArch(),
TreeMap::new,
Collectors.groupingBy((Map.Entry<PkgId, PkgTuple> entry) -> entry.getKey().getName(),
TreeMap::new,
Collectors.groupingBy((Map.Entry<PkgId, PkgTuple> entry) -> entry.getKey().getVersion(),
() -> new TreeMap<>(VersionComparator.getInstance().reversed()),
Collectors.mapping(
Map.Entry::getValue,
CollectionUtils.toUnmodifiableTreeSet(
Comparator.comparing(PkgTuple::getFilename)
)
)
)
)
)
);
builder = Response.ok(result);
builder.tag(etag);
}
builder.cacheControl(cc);
return builder.build();
}
@GET
@Path("hashes")
public Response getHashes() {
EntityManager em = emf.createEntityManager();
TypedQuery<String> query = em.createQuery("SELECT p.md5sum FROM PkgData p", String.class);
return Response.ok(new StringList(query.getResultList())).build();
}
@GET
@Path("files")
public Response getFiles() {
EntityManager em = emf.createEntityManager();
TypedQuery<String> query = em.createQuery("SELECT p.fileName FROM PkgData p", String.class);
return Response.ok(new StringList(query.getResultList())).build();
}
private Response getPackageByHash(String md5sum) {
EntityManager em = emf.createEntityManager();
TypedQuery<PkgData> hquery = em.createNamedQuery("searchByHash", PkgData.class);
if (md5sum != null) hquery.setParameter("md5sum", md5sum);
return manageQueryResult(hquery.getResultList(), true);
}
private Response getPackageByFileName(String file) {
EntityManager em = emf.createEntityManager();
TypedQuery<PkgData> fnquery = em.createNamedQuery("searchByFileName", PkgData.class);
fnquery.setParameter("fileName", file);
return manageQueryResult(fnquery.getResultList(), true);
}
@GET
@Path("filesize/{filename}")
@SneakyThrows
public Response getFileSize(@PathParam("filename")String fileName, @Context Request request) {
CacheControl cc = new CacheControl();
cc.setMaxAge(86400);
cc.setMustRevalidate(true);
cc.setNoCache(true);
EntityTag etag = new EntityTag(Integer.toString(getCachedMap().hashCode()));
Response.ResponseBuilder builder = request.evaluatePreconditions(etag);
if (builder == null) {
java.nio.file.Path res = ctx.getFile(fileName);
if (!Files.exists(res)) throw new NotFoundException(String.format("File '%s' was not found", fileName));
builder = Response.ok(Files.size(res));
builder.tag(etag);
}
builder.cacheControl(cc);
return builder.build();
}
@SneakyThrows
@GET
@Path("download/{filename}")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
public Response downloadPackage(@PathParam("filename") String fileName) {
EntityManager em = emf.createEntityManager();
TypedQuery<PkgData> fnquery = em.createNamedQuery("searchByFileName", PkgData.class);
fnquery.setParameter("fileName", fileName);
try {
PkgData pkg = fnquery.getSingleResult();
StreamingOutput stream = (OutputStream output) -> {
try(InputStream is = Files.newInputStream(ctx.getFile(pkg))) {
JWO.copy(is, output, 0x10000);
}
};
return Response.ok(stream).header("Content-Length", Files.size(ctx.getFile(pkg))).build();
} catch(NoResultException nre) {
throw new NotFoundException();
}
}
@POST
@Path("/doYouWantAny")
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
public Response doYouWantAny(List<String> filenames) {
EntityManager em = emf.createEntityManager();
Set<String> result = filenames.stream().collect(CollectionUtils.toTreeSet());
if (!result.isEmpty()) {
TypedQuery<String> query = em.createQuery("SELECT pkg.fileName from PkgData pkg WHERE pkg.fileName in :filenames", String.class);
query.setParameter("filenames", filenames);
Set<String> toBeRemoved = query.getResultStream().collect(CollectionUtils.toTreeSet());
result.removeAll(toBeRemoved);
return Response.ok(new StringList(result)).build();
}
else {
return Response.ok(result.toArray()).build();
}
}
@POST
@Path("/upload")
@SneakyThrows
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Consumes({"application/x-xz", "application/gzip", "application/x-tar", MediaType.APPLICATION_OCTET_STREAM})
public Response createPackage(
InputStream input,
@MatrixParam("filename") String filename,
@Context UriInfo uriInfo) {
EntityManager em = emf.createEntityManager();
if (filename == null) throw new BadRequestException();
java.nio.file.Path file = Files.createTempFile(ctx.getRepoFolder(), filename, null);
TypedQuery<PkgData> fquery = em.createNamedQuery("searchByFileName", PkgData.class);
fquery.setParameter("fileName", filename);
List<PkgData> savedFiles = fquery.getResultList();
Response result;
if (savedFiles.size() > 0) result = Response.notModified().build();
else {
try(OutputStream output = Files.newOutputStream(file)) {
JWO.copy(input, output, 0x10000);
PkgData pkg = PkgDataImpl.parseFile(file,
CompressionFormatImpl.guess(Paths.get(filename)));
pkg.setFileName(filename);
Optional.ofNullable(em.find(PkgData.class, pkg.getId())).ifPresent((Con<PkgData>) (pkgData -> {
em.remove(pkgData);
Files.delete(ctx.getRepoFolder().resolve(pkgData.getFileName()));
}));
log.info("Persisting package {}", pkg.getFileName());
em.persist(pkg);
URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.getFileName()).build();
Files.move(file, ctx.getRepoFolder().resolve(filename), StandardCopyOption.ATOMIC_MOVE);
ctx.getInvalidateCache().set(true);
cachedMap = null;
result = Response.created(pkgUri).build();
} catch (Throwable t) {
Files.delete(file);
throw t;
}
}
return result;
}
@GET
@Path("/search")
public List<PkgData> searchPackage(
@QueryParam("name") String name,
@QueryParam("version") String version,
@QueryParam("arch") String arch,
@QueryParam("page") int pageNumber,
@QueryParam("pageSize") int pageSize,
@QueryParam("fileName") String fileName) {
return service.searchPackage(name, version, arch, pageNumber, pageSize, fileName);
}
@OPTIONS
@Path("/downloadTar")
@Produces("text/plain; charset=UTF-8")
public Response options() {
return Response.ok("POST, OPTIONS").build();
}
@POST
@Path("/downloadTar")
@Produces("application/x-tar")
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
public Response downloadTar(@FormParam("pkgs") String formData) {
String[] files = URLDecoder.decode(formData, StandardCharsets.UTF_8).split(" ");
Arrays.stream(files)
.filter(fileName -> !Files.exists(ctx.getFile(fileName)))
.forEach(fileName -> {
throw JWO.newThrowable(NotFoundException.class, "Package file '{}' does not exist", fileName);
});
StreamingOutput stream = new StreamingOutput() {
private final byte[] buffer = new byte[0x10000];
@Override
@SneakyThrows
public void write(OutputStream output) {
try(TarArchiveOutputStream taos = new TarArchiveOutputStream(output)) {
for (String fname : files) {
log.info(fname);
java.nio.file.Path file = ctx.getFile(fname);
try(InputStream input = Files.newInputStream(file)) {
TarArchiveEntry entry = new TarArchiveEntry(fname);
entry.setSize(Files.size(file));
taos.putArchiveEntry(entry);
JWO.copy(input, taos, buffer);
taos.closeArchiveEntry();
}
}
} catch (Exception ex) {
log.error(ex.getMessage(), ex);
throw ex;
}
}
};
return Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build();
}
}

17
src/main/java/net/woggioni/jpacrepo/service/WxRsApplication.java Normal file
View File

@@ -0,0 +1,17 @@
package net.woggioni.jpacrepo.service;
import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
import java.util.HashSet;
import java.util.Set;
@ApplicationPath("api")
public class WxRsApplication extends Application {
@Override
public Set<Class<?>> getClasses() {
Set<Class<?>> result = new HashSet<>();
result.add(PacmanWebService.class);
return result;
}
}

31
src/main/java/net/woggioni/jpacrepo/service/wire/PkgDataList.java Normal file
View File

@@ -0,0 +1,31 @@
package net.woggioni.jpacrepo.service.wire;
import net.woggioni.jpacrepo.api.model.PkgData;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import java.util.List;
@XmlRootElement
public class PkgDataList extends ArrayList<PkgData> {
public PkgDataList(List<PkgData> l) {
for (PkgData el : l) add(el);
}
public PkgDataList(PkgData... elements) {
for (PkgData el : elements) add(el);
}
@XmlElement(name = "pkgData")
List<PkgData> getItems() {
return this;
}
void setItems(List<PkgData> pkgs) {
this.clear();
this.addAll(pkgs);
}
}

15
src/main/java/net/woggioni/jpacrepo/service/wire/PkgTuple.java Normal file
View File

@@ -0,0 +1,15 @@
package net.woggioni.jpacrepo.service.wire;
import lombok.Data;
import javax.xml.bind.annotation.XmlRootElement;
@Data
@XmlRootElement
public class PkgTuple {
String md5sum;
String filename;
long size;
}

28
src/main/java/net/woggioni/jpacrepo/service/wire/StringList.java Normal file
View File

@@ -0,0 +1,28 @@
package net.woggioni.jpacrepo.service.wire;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import java.util.List;
@XmlRootElement
public class StringList extends ArrayList<String> {
public StringList(Iterable<String> l) {
for (String el : l) add(el);
}
public StringList(String... strings) {
for (String el : strings) add(el);
}
@XmlElement(name = "pkgData")
List<String> getItems() {
return this;
}
void setItems(List<String> strings) {
this.clear();
this.addAll(strings);
}
}

225
src/main/java/net/woggioni/jpacrepo/servlet/AbstractFileServlet.java
View File

@@ -1,12 +1,16 @@
package net.woggioni.jpacrepo.servlet;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
@@ -97,8 +101,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
* @author Bauke Scholtz
* @since 2.2
*/
public abstract class AbstractFileServlet extends HttpServlet
{
public abstract class AbstractFileServlet extends HttpServlet {
// Constants ------------------------------------------------------------------------------------------------------
@@ -113,16 +116,13 @@ public abstract class AbstractFileServlet extends HttpServlet
private static final String CONTENT_DISPOSITION_HEADER = "%s;filename=\"%2$s\"; filename*=UTF-8''%2$s";
private static final String MULTIPART_BOUNDARY = UUID.randomUUID().toString();
private static long stream(InputStream input, OutputStream output) throws IOException
{
private static long stream(InputStream input, OutputStream output) throws IOException {
try (ReadableByteChannel inputChannel = Channels.newChannel(input);
WritableByteChannel outputChannel = Channels.newChannel(output))
{
WritableByteChannel outputChannel = Channels.newChannel(output)) {
ByteBuffer buffer = ByteBuffer.allocateDirect(DEFAULT_STREAM_BUFFER_SIZE);
long size = 0;
while (inputChannel.read(buffer) != -1)
{
while (inputChannel.read(buffer) != -1) {
buffer.flip();
size += outputChannel.write(buffer);
buffer.clear();
@@ -145,32 +145,26 @@ public abstract class AbstractFileServlet extends HttpServlet
* @throws IOException When an I/O error occurs.
* @since 2.2
*/
public static long stream(File file, OutputStream output, long start, long length) throws IOException
{
if (start == 0 && length >= file.length())
{
public static long stream(File file, OutputStream output, long start, long length) throws IOException {
if (start == 0 && length >= file.length()) {
return stream(new FileInputStream(file), output);
}
try (FileChannel fileChannel = (FileChannel) Files.newByteChannel(file.toPath(), StandardOpenOption.READ))
{
try (FileChannel fileChannel = (FileChannel) Files.newByteChannel(file.toPath(), StandardOpenOption.READ)) {
WritableByteChannel outputChannel = Channels.newChannel(output);
ByteBuffer buffer = ByteBuffer.allocateDirect(DEFAULT_STREAM_BUFFER_SIZE);
long size = 0;
while (fileChannel.read(buffer, start + size) != -1)
{
while (fileChannel.read(buffer, start + size) != -1) {
buffer.flip();
if (size + buffer.limit() > length)
{
if (size + buffer.limit() > length) {
buffer.limit((int) (length - size));
}
size += outputChannel.write(buffer);
if (size >= length)
{
if (size >= length) {
break;
}
@@ -182,27 +176,20 @@ public abstract class AbstractFileServlet extends HttpServlet
}
private static String encodeURL(String string)
{
if (string == null)
{
private static String encodeURL(String string) {
if (string == null) {
return null;
}
try
{
try {
return URLEncoder.encode(string, UTF_8.name());
}
catch (UnsupportedEncodingException e)
{
} catch (UnsupportedEncodingException e) {
throw new UnsupportedOperationException(ERROR_UNSUPPORTED_ENCODING, e);
}
}
private static String encodeURI(String string)
{
if (string == null)
{
private static String encodeURI(String string) {
if (string == null) {
return null;
}
@@ -218,74 +205,60 @@ public abstract class AbstractFileServlet extends HttpServlet
// Actions --------------------------------------------------------------------------------------------------------
@Override
protected void doHead(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
protected void doHead(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
doRequest(request, response, true);
}
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
doRequest(request, response, false);
}
private void doRequest(HttpServletRequest request, HttpServletResponse response, boolean head) throws IOException
{
private void doRequest(HttpServletRequest request, HttpServletResponse response, boolean head) throws IOException {
response.reset();
Resource resource;
try
{
try {
resource = new Resource(getFile(request));
}
catch (IllegalArgumentException e)
{
} catch (IllegalArgumentException e) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST);
return;
}
if (resource.file == null)
{
if (resource.file == null) {
handleFileNotFound(request, response);
return;
}
if (preconditionFailed(request, resource))
{
if (preconditionFailed(request, resource)) {
response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED);
return;
}
setCacheHeaders(response, resource, getExpireTime(request, resource.file));
if (notModified(request, resource))
{
if (notModified(request, resource)) {
response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
return;
}
List<Range> ranges = getRanges(request, resource);
if (ranges == null)
{
if (ranges == null) {
response.setHeader("Content-Range", "bytes */" + resource.length);
response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
return;
}
if (!ranges.isEmpty())
{
if (!ranges.isEmpty()) {
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
}
else
{
} else {
ranges.add(new Range(0, resource.length - 1)); // Full content.
}
String contentType = setContentHeaders(request, response, resource, ranges);
if (head)
{
if (head) {
return;
}
@@ -315,8 +288,7 @@ public abstract class AbstractFileServlet extends HttpServlet
* @throws IOException When something fails at I/O level.
* @since 2.3
*/
protected void handleFileNotFound(HttpServletRequest request, HttpServletResponse response) throws IOException
{
protected void handleFileNotFound(HttpServletRequest request, HttpServletResponse response) throws IOException {
response.sendError(HttpServletResponse.SC_NOT_FOUND);
}
@@ -329,8 +301,7 @@ public abstract class AbstractFileServlet extends HttpServlet
* @param file The involved file.
* @return The client cache expire time in seconds (not milliseconds!).
*/
protected long getExpireTime(HttpServletRequest request, File file)
{
protected long getExpireTime(HttpServletRequest request, File file) {
return DEFAULT_EXPIRE_TIME_IN_SECONDS;
}
@@ -344,15 +315,11 @@ public abstract class AbstractFileServlet extends HttpServlet
* @param file The involved file.
* @return The content type associated with the given HTTP servlet request and file.
*/
protected String getContentType(HttpServletRequest request, File file)
{
protected String getContentType(HttpServletRequest request, File file) {
String type = request.getServletContext().getMimeType(file.getName());
if (type != null)
{
if (type != null) {
return type;
}
else
{
} else {
return "application/octet-stream";
}
}
@@ -370,8 +337,7 @@ public abstract class AbstractFileServlet extends HttpServlet
* @return <code>true</code> if we must force a "Save As" dialog based on the given HTTP servlet request and content
* type.
*/
protected boolean isAttachment(HttpServletRequest request, String contentType)
{
protected boolean isAttachment(HttpServletRequest request, String contentType) {
String accept = request.getHeader("Accept");
return (!contentType.startsWith("text") && !contentType.startsWith("image")) && (accept == null || !accepts(accept, contentType));
}
@@ -387,8 +353,7 @@ public abstract class AbstractFileServlet extends HttpServlet
* @return The file name to be used in <code>Content-Disposition</code> header.
* @since 2.3
*/
protected String getAttachmentName(HttpServletRequest request, File file)
{
protected String getAttachmentName(HttpServletRequest request, File file) {
return file.getName();
}
@@ -397,8 +362,7 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Returns true if it's a conditional request which must return 412.
*/
private boolean preconditionFailed(HttpServletRequest request, Resource resource)
{
private boolean preconditionFailed(HttpServletRequest request, Resource resource) {
String match = request.getHeader("If-Match");
long unmodified = request.getDateHeader("If-Unmodified-Since");
return (match != null) ? !matches(match, resource.eTag) : (unmodified != -1 && modified(unmodified, resource.lastModified));
@@ -407,8 +371,7 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Set cache headers.
*/
private void setCacheHeaders(HttpServletResponse response, Resource resource, long expires)
{
private void setCacheHeaders(HttpServletResponse response, Resource resource, long expires) {
//Servlets.setCacheHeaders(response, expires);
response.setHeader("ETag", resource.eTag);
response.setDateHeader("Last-Modified", resource.lastModified);
@@ -417,8 +380,7 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Returns true if it's a conditional request which must return 304.
*/
private boolean notModified(HttpServletRequest request, Resource resource)
{
private boolean notModified(HttpServletRequest request, Resource resource) {
String noMatch = request.getHeader("If-None-Match");
long modified = request.getDateHeader("If-Modified-Since");
return (noMatch != null) ? matches(noMatch, resource.eTag) : (modified != -1 && !modified(modified, resource.lastModified));
@@ -427,45 +389,34 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Get requested ranges. If this is null, then we must return 416. If this is empty, then we must return full file.
*/
private List<Range> getRanges(HttpServletRequest request, Resource resource)
{
private List<Range> getRanges(HttpServletRequest request, Resource resource) {
List<Range> ranges = new ArrayList<>(1);
String rangeHeader = request.getHeader("Range");
if (rangeHeader == null)
{
if (rangeHeader == null) {
return ranges;
}
else if (!RANGE_PATTERN.matcher(rangeHeader).matches())
{
} else if (!RANGE_PATTERN.matcher(rangeHeader).matches()) {
return null; // Syntax error.
}
String ifRange = request.getHeader("If-Range");
if (ifRange != null && !ifRange.equals(resource.eTag))
{
try
{
if (ifRange != null && !ifRange.equals(resource.eTag)) {
try {
long ifRangeTime = request.getDateHeader("If-Range");
if (ifRangeTime != -1 && modified(ifRangeTime, resource.lastModified))
{
if (ifRangeTime != -1 && modified(ifRangeTime, resource.lastModified)) {
return ranges;
}
}
catch (IllegalArgumentException ifRangeHeaderIsInvalid)
{
} catch (IllegalArgumentException ifRangeHeaderIsInvalid) {
return ranges;
}
}
for (String rangeHeaderPart : rangeHeader.split("=")[1].split(","))
{
for (String rangeHeaderPart : rangeHeader.split("=")[1].split(",")) {
Range range = parseRange(rangeHeaderPart, resource.length);
if (range == null)
{
if (range == null) {
return null; // Logic error.
}
@@ -478,23 +429,18 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Parse range header part. Returns null if there's a logic error (i.e. start after end).
*/
private Range parseRange(String range, long length)
{
private Range parseRange(String range, long length) {
long start = sublong(range, 0, range.indexOf('-'));
long end = sublong(range, range.indexOf('-') + 1, range.length());
if (start == -1)
{
if (start == -1) {
start = length - end;
end = length - 1;
}
else if (end == -1 || end > length - 1)
{
} else if (end == -1 || end > length - 1) {
end = length - 1;
}
if (start > end)
{
if (start > end) {
return null; // Logic error.
}
@@ -504,27 +450,22 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Set content headers.
*/
private String setContentHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, List<Range> ranges)
{
private String setContentHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, List<Range> ranges) {
String contentType = getContentType(request, resource.file);
String disposition = isAttachment(request, contentType) ? "attachment" : "inline";
String filename = encodeURI(getAttachmentName(request, resource.file));
response.setHeader("Content-Disposition", String.format(CONTENT_DISPOSITION_HEADER, disposition, filename));
response.setHeader("Accept-Ranges", "bytes");
if (ranges.size() == 1)
{
if (ranges.size() == 1) {
Range range = ranges.get(0);
response.setContentType(contentType);
response.setHeader("Content-Length", String.valueOf(range.length));
if (response.getStatus() == HttpServletResponse.SC_PARTIAL_CONTENT)
{
if (response.getStatus() == HttpServletResponse.SC_PARTIAL_CONTENT) {
response.setHeader("Content-Range", "bytes " + range.start + "-" + range.end + "/" + resource.length);
}
}
else
{
} else {
response.setContentType("multipart/byteranges; boundary=" + MULTIPART_BOUNDARY);
}
@@ -534,19 +475,14 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Write given file to response with given content type and ranges.
*/
private void writeContent(HttpServletResponse response, Resource resource, List<Range> ranges, String contentType) throws IOException
{
private void writeContent(HttpServletResponse response, Resource resource, List<Range> ranges, String contentType) throws IOException {
ServletOutputStream output = response.getOutputStream();
if (ranges.size() == 1)
{
if (ranges.size() == 1) {
Range range = ranges.get(0);
stream(resource.file, output, range.start, range.length);
}
else
{
for (Range range : ranges)
{
} else {
for (Range range : ranges) {
output.println();
output.println("--" + MULTIPART_BOUNDARY);
output.println("Content-Type: " + contentType);
@@ -564,8 +500,7 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Returns true if the given match header matches the given ETag value.
*/
private static boolean matches(String matchHeader, String eTag)
{
private static boolean matches(String matchHeader, String eTag) {
String[] matchValues = matchHeader.split("\\s*,\\s*");
Arrays.sort(matchValues);
return Arrays.binarySearch(matchValues, eTag) > -1
@@ -575,8 +510,7 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Returns true if the given modified header is older than the given last modified value.
*/
private static boolean modified(long modifiedHeader, long lastModified)
{
private static boolean modified(long modifiedHeader, long lastModified) {
return (modifiedHeader + ONE_SECOND_IN_MILLIS <= lastModified); // That second is because the header is in seconds, not millis.
}
@@ -584,8 +518,7 @@ public abstract class AbstractFileServlet extends HttpServlet
* Returns a substring of the given string value from the given begin index to the given end index as a long.
* If the substring is empty, then -1 will be returned.
*/
private static long sublong(String value, int beginIndex, int endIndex)
{
private static long sublong(String value, int beginIndex, int endIndex) {
String substring = value.substring(beginIndex, endIndex);
return substring.isEmpty() ? -1 : Long.parseLong(substring);
}
@@ -593,8 +526,7 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Returns true if the given accept header accepts the given value.
*/
private static boolean accepts(String acceptHeader, String toAccept)
{
private static boolean accepts(String acceptHeader, String toAccept) {
String[] acceptValues = acceptHeader.split("\\s*(,|;)\\s*");
Arrays.sort(acceptValues);
return Arrays.binarySearch(acceptValues, toAccept) > -1
@@ -607,24 +539,19 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Convenience class for a file resource.
*/
private static class Resource
{
private static class Resource {
private final File file;
private final long length;
private final long lastModified;
private final String eTag;
public Resource(File file)
{
if (file != null && file.isFile())
{
public Resource(File file) {
if (file != null && file.isFile()) {
this.file = file;
length = file.length();
lastModified = file.lastModified();
eTag = String.format(ETAG, encodeURL(file.getName()), lastModified);
}
else
{
} else {
this.file = null;
length = 0;
lastModified = 0;
@@ -637,14 +564,12 @@ public abstract class AbstractFileServlet extends HttpServlet
/**
* Convenience class for a byte range.
*/
private static class Range
{
private static class Range {
private final long start;
private final long end;
private final long length;
public Range(long start, long end)
{
public Range(long start, long end) {
this.start = start;
this.end = end;
length = end - start + 1;

8
src/main/java/net/woggioni/jpacrepo/servlet/FileServlet.java
View File

@@ -9,18 +9,16 @@ import java.io.File;
import java.nio.file.Path;
@WebServlet("/archive/*")
public class FileServlet extends AbstractFileServlet
{
private Path root;
public class FileServlet extends AbstractFileServlet {
private final Path root;
@Inject
public FileServlet(AppConfig ctx){
root = ctx.repoFolder();
root = ctx.getRepoFolder();
}
@Override
protected File getFile(HttpServletRequest request) throws IllegalArgumentException {
return root.resolve(request.getPathInfo().substring(1)).toFile();
}
}

26
src/main/java/net/woggioni/jpacrepo/version/PkgIdComparator.java
View File

@@ -1,21 +1,19 @@
package net.woggioni.jpacrepo.version;
import net.woggioni.jpacrepo.pacbase.PkgId;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.NoArgsConstructor;
import net.woggioni.jpacrepo.api.model.PkgId;
import java.util.Comparator;
public class PkgIdComparator implements Comparator<PkgId> {
private final Comparator<PkgId> comparator;
public PkgIdComparator() {
VersionComparator vc = new VersionComparator();
comparator = Comparator.comparing(PkgId::name)
.thenComparing(PkgId::version, vc)
.thenComparing(PkgId::arch);
}
@Override
public int compare(PkgId id1, PkgId id2) {
return comparator.compare(id1, id2);
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class PkgIdComparator {
@Getter
private static final Comparator<PkgId> comparator;
static {
comparator = Comparator.comparing(PkgId::getName)
.thenComparing(PkgId::getVersion, VersionComparator.getInstance())
.thenComparing(PkgId::getArch);
}
}

14
src/main/java/net/woggioni/jpacrepo/version/VersionComparator.java
View File

@@ -1,6 +1,10 @@
package net.woggioni.jpacrepo.version;
import com.sun.jna.Native;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.util.Comparator;
@@ -13,10 +17,12 @@ class AlpmLibrary {
}
}
public class VersionComparator implements Comparator<String> {
@Override
public int compare(String version1, String version2) {
return AlpmLibrary.alpm_pkg_vercmp(version1, version2);
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class VersionComparator {
@Getter
private static final Comparator<String> instance;
static {
instance = AlpmLibrary::alpm_pkg_vercmp;
}
}

24
src/main/resources/META-INF/persistence.xml Normal file
View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<persistence xmlns="http://xmlns.jcp.org/xml/ns/persistence"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/persistence http://xmlns.jcp.org/xml/ns/persistence/persistence_2_1.xsd"
version="2.1">
<persistence-unit name="jpacrepo_pu" transaction-type="JTA">
<class>net.woggioni.jpacrepo.api.model.PkgData</class>
<class>net.woggioni.jpacrepo.api.model.PkgId</class>
<exclude-unlisted-classes>true</exclude-unlisted-classes>
<properties>
<property name="org.jboss.logging.provider" value="log4j2"/>
<property name="javax.persistence.schema-generation.create-source" value="script-then-metadata"/>
<property name="javax.persistence.schema-generation.create-script-source"
value="META-INF/sql/CreateSchema.sql"/>
<property name="javax.persistence.schema-generation.drop-source" value="metadata-then-script"/>
<property name="javax.persistence.schema-generation.drop-script-source"
value="META-INF/sql/DropSchema.sql"/>
<property name="hibernate.default_schema" value="jpacrepo"/>
<property name="hibernate.dialect" value="org.hibernate.dialect.PostgreSQL82Dialect"/>
</properties>
</persistence-unit>
</persistence>

41
src/main/scala/net/woggioni/jpacrepo/config/AppConfig.scala
View File

@@ -1,41 +0,0 @@
package net.woggioni.jpacrepo.config
import java.nio.file.{Files, Path, Paths}
import java.util.Properties
import java.util.concurrent.atomic.AtomicBoolean
import net.woggioni.jpacrepo.pacbase.PkgData
import net.woggioni.jpacrepo.persistence.InitialSchemaAction
import net.woggioni.jpacrepo.utils.Utils._
object AppConfig {
def apply(propertyFile: String): AppConfig = {
val getProperty = new Properties().let { it =>
val path = Paths.get(propertyFile)
if (Files.exists(path)) {
Files.newInputStream(path).use { is =>
it.load(is)
}
}
(key : String, default : String) => System.getProperty(s"net.woggioni.jpacrepo.$key") match {
case null => it.getProperty(key, default)
case path: String => path
}
}
new AppConfig(
repoFolder = getProperty("RepoFolder", "net.woggioni.jpacrepo.RepoFolder")
.let { it => Paths.get(it) },
initialSchemaAction = getProperty("InitialSchemaAction", "none").let(InitialSchemaAction(_)))
}
}
case class AppConfig(val repoFolder: Path,
val initialSchemaAction: InitialSchemaAction) {
var invalidateCache = new AtomicBoolean(true)
def getFile(pkg: PkgData) = repoFolder.resolve(pkg.fileName)
def getFile(fileName: String) = repoFolder.resolve(fileName)
}

5
src/main/scala/net/woggioni/jpacrepo/exception/Exceptions.scala
View File

@@ -1,5 +0,0 @@
package net.woggioni.jpacrepo.exception
class ParseException(msg : String, cause : Throwable) extends RuntimeException(msg, cause) {
def this(msg : String) = this(msg, null)
}

48
src/main/scala/net/woggioni/jpacrepo/factory/BeanFactory.scala
View File

@@ -1,48 +0,0 @@
package net.woggioni.jpacrepo.factory
import java.util.Properties
import javax.annotation.PostConstruct
import javax.enterprise.inject.Produces
import javax.enterprise.inject.spi.InjectionPoint
import javax.faces.bean.ApplicationScoped
import javax.inject.Inject
import javax.persistence.{EntityManagerFactory, Persistence}
import net.woggioni.jpacrepo.config.AppConfig
import org.slf4j.LoggerFactory
class PersistenceUnitFactory {
@Inject
private var appConfig : AppConfig = _
private var emf : EntityManagerFactory = _
@PostConstruct
def init(): Unit = {
val properties = new Properties()
properties.put("javax.persistence.schema-generation.database.action", appConfig.initialSchemaAction.value)
emf = Persistence.createEntityManagerFactory("jpacrepo_pu", properties)
}
@Produces
private def createEntityManagerFactory = emf
}
class BeanFactory {
@Produces
@ApplicationScoped
def produce: AppConfig = {
val ctx = AppConfig(
System.getProperty("net.woggioni.jpacrepo.configuration.file",
"/etc/jpacrepo/server.properties"))
ctx
}
@Produces
private def createLogger(injectionPoint: InjectionPoint) =
LoggerFactory.getLogger(injectionPoint.getMember.getDeclaringClass.getName)
}

46
src/main/scala/net/woggioni/jpacrepo/model/Hasher.scala
View File

@@ -1,46 +0,0 @@
package net.woggioni.jpacrepo.model
import java.io.InputStream
import java.security.MessageDigest
object Hasher {
private val md5 = MessageDigest.getInstance("MD5")
private def getHash(md : MessageDigest, is: InputStream): Array[Byte] = {
val buffer = new Array[Byte](4096)
var read = 0
while ( {
read = is.read(buffer, 0, buffer.length)
read >= 0
}) {
md.update(buffer, 0, read)
}
md.digest
}
def computeMD5(is: InputStream): String = Hasher.bytesToHex(getHash(md5, is))
private val hexArray = "0123456789ABCDEF".toCharArray
def bytesToHex(bytes: Array[Byte]): String = {
val hexChars = new Array[Char](bytes.length * 2)
var j = 0
while (j < bytes.length) {
val v = bytes(j) & 0xFF
hexChars(j * 2) = hexArray(v >>> 4)
hexChars(j * 2 + 1) = hexArray(v & 0x0F)
j += 1
}
new String(hexChars)
}
}
class Hasher(val algorithm: String) {
private val md = MessageDigest.getInstance(algorithm)
def getHash(is : InputStream) = Hasher.getHash(md, is)
def getHashString(is: InputStream): String = Hasher.bytesToHex(getHash(is))
}

11
src/main/scala/net/woggioni/jpacrepo/model/MD5InputStream.scala
View File

@@ -1,11 +0,0 @@
package net.woggioni.jpacrepo.model
import java.io.InputStream
import java.security.{DigestInputStream, MessageDigest}
class MD5InputStream(val is: InputStream) extends DigestInputStream(is, MessageDigest.getInstance("md5")) {
def digest(): String = {
val digest = getMessageDigest.digest
Hasher.bytesToHex(digest)
}
}

114
src/main/scala/net/woggioni/jpacrepo/model/Parser.scala
View File

@@ -1,114 +0,0 @@
package net.woggioni.jpacrepo.model
import java.io.{BufferedInputStream, InputStream}
import java.nio.file.{Files, Path}
import java.util.Date
import java.util.zip.GZIPInputStream
import net.woggioni.jpacrepo.exception.ParseException
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData, PkgId}
import net.woggioni.jpacrepo.utils.Utils._
import net.woggioni.jzstd.ZstdInputStream
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream
import scala.io.Source
import scala.jdk.CollectionConverters._
object Parser {
def parseFile(file: Path, compressionFormat : CompressionFormat): PkgData = {
val hasher = new Hasher("MD5")
val decompressorStreamConstructor = compressionFormat match {
case CompressionFormat.XZ => arg : InputStream => new XZCompressorInputStream(arg)
case CompressionFormat.Z_STANDARD => is : InputStream => ZstdInputStream.from(is)
case CompressionFormat.GZIP => is : InputStream => new GZIPInputStream(is)
case format => throw new ParseException(s"Unsupported compression format '$format'")
}
val is = new TarArchiveInputStream(
decompressorStreamConstructor(
new BufferedInputStream(
Files.newInputStream(file))))
try {
var archiveEntry = is.getNextEntry
while (archiveEntry != null) {
if (archiveEntry.getName == ".PKGINFO") {
val buffer = new Array[Byte](archiveEntry.getSize.toInt)
is.read(buffer)
val metadata = Source.fromBytes(buffer)
.getLines()
.map(_.trim)
.filter(!_.isEmpty)
.filter(!_.startsWith("#"))
.map(line => {
val equals = line.indexOf("=")
if (equals < 0) {
throw new ParseException(s"Error parsing .PKGINFO file in '${file}'")
}
else {
(line.substring(0, equals).trim, line.substring(equals + 1, line.length).trim)
}
})
.to(LazyList)
.groupBy(_._1)
.map(pair => pair._1 -> pair._2.map(_._2).toList)
val data = new PkgData
data.id = new PkgId
for (pair <- metadata) {
val (key, value) = pair
key match {
case "size" =>
data.size = value.head.toLong
case "arch" =>
data.id.arch = value.head
case "replaces" =>
data.replaces = value.toSet.asJava
case "packager" =>
data.packager = value.head
case "url" =>
data.url = value.head
case "pkgname" =>
data.id.name = value.head
case "builddate" =>
data.buildDate = new Date(value.head.toLong * 1000)
case "license" =>
data.license = value.head
case "pkgver" =>
data.id.version = value.head
case "pkgdesc" =>
data.description = value.head
case "provides" =>
data.provides = value.toSet.asJava
case "conflict" =>
data.conflict = value.toSet.asJava
case "backup" =>
data.backup = value.toSet.asJava
case "optdepend" =>
data.optdepend = value.toSet.asJava
case "depend" =>
data.depend = value.toSet.asJava
case "makedepend" =>
data.makedepend = value.toSet.asJava
case "makepkgopt" =>
data.makeopkgopt = value.toSet.asJava
case "pkgbase" =>
data.base = value.head
case _ =>
}
}
data.md5sum = Files.newInputStream(file).use(hasher.getHashString)
data.fileName = file.getFileName.toString
return data
}
else {
archiveEntry = is.getNextEntry
}
}
throw new ParseException(s".PKGINFO file not found in '$file'")
} finally {
is.close()
}
}
}

32
src/main/scala/net/woggioni/jpacrepo/pacbase/CompressionFormat.scala
View File

@@ -1,32 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import java.nio.file.Path
import net.woggioni.jpacrepo.exception.ParseException
import net.woggioni.jwo.JWO
sealed class CompressionFormat(val extension : String) {
override def toString: String = extension
}
object CompressionFormat {
private val map = LazyList(XZ, GZIP, Z_STANDARD)
.map(v => v.extension -> v).toMap
def apply(text : String) : Option[CompressionFormat] = map.get(text)
def values() = map.values
def guess(file : Path) = {
val extension = JWO.splitExtension(file).orElseThrow(() =>
new ParseException(s"Unable to parse file extension for '${file.getFileName}'")
)._2
CompressionFormat(extension.substring(1)) match {
case Some(compressionFormat) => compressionFormat
case None => throw new IllegalArgumentException(s"Unknown compression format for file extension '$extension'")
}
}
case object XZ extends CompressionFormat("xz")
case object GZIP extends CompressionFormat("gz")
case object Z_STANDARD extends CompressionFormat("zst")
}

8
src/main/scala/net/woggioni/jpacrepo/pacbase/Ordering.scala
View File

@@ -1,8 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import net.woggioni.jpacrepo.version.VersionComparator
class VersionOrdering extends Ordering[String] {
private val comparator = new VersionComparator
override def compare(x: String, y: String): Int = comparator.compare(x, y)
}

78
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgData.scala
View File

@@ -1,78 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import java.util
import java.util.Date
import javax.persistence._
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlRootElement}
@Entity
@Access(AccessType.FIELD)
@NamedQueries(value = Array[NamedQuery](
new NamedQuery(name = "searchByFileName", query = "SELECT p FROM PkgData p WHERE p.fileName = :fileName"),
new NamedQuery(name = "searchByName", query = "SELECT p FROM PkgData p WHERE p.id.name = :name"),
new NamedQuery(name = "searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id"),
new NamedQuery(name = "searchByHash", query = "SELECT p FROM PkgData p WHERE p.md5sum = :md5sum")
))
@Table(indexes = Array(
new Index(columnList = "md5sum", unique = true),
new Index(columnList = "fileName", unique = true))
)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
class PkgData {
@EmbeddedId
var id: PkgId = _
var base: String = _
var description: String = _
var url: String = _
@Temporal(TemporalType.TIMESTAMP)
var buildDate: Date = _
var packager: String = _
var size = 0L
var license: String = _
var md5sum: String = _
var fileName: String = _
@ElementCollection(fetch = FetchType.EAGER)
var replaces: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var conflict: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var provides: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var depend: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var optdepend: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var makedepend: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var makeopkgopt: util.Set[String] = _
@ElementCollection(fetch = FetchType.EAGER)
var backup: util.Set[String] = _
@Temporal(TemporalType.TIMESTAMP)
var updTimestamp: Date = _
@PreUpdate
@PrePersist
private def writeTimestamp(): Unit = updTimestamp = new Date
}

31
src/main/scala/net/woggioni/jpacrepo/pacbase/PkgId.scala
View File

@@ -1,31 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import javax.persistence.{Access, AccessType, Embeddable}
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlRootElement}
@Embeddable
@Access(AccessType.FIELD)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
class PkgId extends Serializable {
var name : String = null
var version : String = null
var arch : String = null
override def equals(obj: Any): Boolean = {
obj match {
case null => false
case pkgId : PkgId => name == pkgId.name && version == pkgId.version && arch == pkgId.arch
}
}
override def hashCode(): Int = {
var result : Int = 0
if(name != null) result ^= name.hashCode
if(version != null) result ^= version.hashCode
if(arch != null) result ^= arch.hashCode
result
}
}

19
src/main/scala/net/woggioni/jpacrepo/persistence/InitialSchemaAction.scala
View File

@@ -1,19 +0,0 @@
package net.woggioni.jpacrepo.persistence
sealed class InitialSchemaAction(val value : String) {
override def toString: String = value
}
object InitialSchemaAction {
private val map = LazyList(NONE, CREATE, DROP, DROP_AND_CREATE)
.map(v => v.value -> v).toMap
def apply(text : String) : InitialSchemaAction = map(text)
def values() = map.values
case object NONE extends InitialSchemaAction("none")
case object CREATE extends InitialSchemaAction("create")
case object DROP extends InitialSchemaAction("drop")
case object DROP_AND_CREATE extends InitialSchemaAction("drop-and-create")
}

174
src/main/scala/net/woggioni/jpacrepo/service/PacmanServiceEJB.scala
View File

@@ -1,174 +0,0 @@
package net.woggioni.jpacrepo.service
import java.nio.file.{Files, Path}
import java.util
import java.util.{Calendar, Date}
import javax.annotation.PostConstruct
import javax.ejb._
import javax.inject.Inject
import javax.persistence._
import net.woggioni.jpacrepo.config.AppConfig
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData}
import net.woggioni.jpacrepo.persistence.QueryEngine
import org.slf4j.Logger
import scala.jdk.CollectionConverters._
@Remote
trait PacmanServiceRemote {
def syncDB(): Unit
def deletePackage(filename: String): Unit
}
@Local
trait PacmanServiceView extends PacmanServiceRemote {
def countResults(name: String, version: String, arch: String): Long
def searchPackage(name: String, version: String, arch: String, page: Int, pageSize: Int, fileName: String): util.List[PkgData]
}
@Startup
@Singleton
@Lock(LockType.READ)
@TransactionManagement(TransactionManagementType.CONTAINER)
@ConcurrencyManagement(ConcurrencyManagementType.CONTAINER)
@Local(Array(classOf[PacmanServiceView]))
@Remote(Array(classOf[PacmanServiceRemote]))
class PacmanServiceEJB extends PacmanServiceView {
@Inject
private var emf: EntityManagerFactory = _
@Inject
private var ctx: AppConfig = _
@Inject
private var logger: Logger = _
final private val nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name"
final private val hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"
final private val hashQueryCount = "SELECT count(pdata) FROM PkgData pdata WHERE md5sum = :md5sum"
private def deletePkgData(em: EntityManager, pkgData: PkgData): Unit = {
if (pkgData.depend != null) pkgData.depend.asScala.foreach(em.remove)
if (pkgData.replaces != null) pkgData.replaces.asScala.foreach(em.remove)
if (pkgData.conflict != null) pkgData.conflict.asScala.foreach(em.remove)
if (pkgData.provides != null) pkgData.provides.asScala.foreach(em.remove)
if (pkgData.makedepend != null) pkgData.makedepend.asScala.foreach(em.remove)
if (pkgData.makeopkgopt != null) pkgData.makeopkgopt.asScala.foreach(em.remove)
if (pkgData.backup != null) pkgData.backup.asScala.foreach(em.remove)
em.remove(pkgData)
}
@Asynchronous
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Schedule(hour = "4", minute = "00", persistent = false)
override def syncDB() = {
val em = emf.createEntityManager
logger.info("Starting repository cleanup")
//Elimina i pacchetti sul DB che non esistono più nel filesystem
logger.info("Searching for packages that are no more in the filesystem")
val listaDB = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String]).getResultList
logger.info("Got list of filenames from db")
val knownPkg = listaDB
.asScala
.filter(fileName => {
val file = ctx.getFile(fileName)
val result = Files.exists(file)
if (!result) {
logger.info(s"Removing package ${file.getFileName} which was not found in filesystem")
em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
.setParameter("fileName", file.getFileName)
.getResultList.asScala.foreach((pkgData: PkgData) => deletePkgData(em, pkgData))
}
result
}).toSet
logger.info("Searching for new packages or packages that were modified after being added to the database")
Files.list(ctx.repoFolder).iterator().asScala.filter(file => {
val name = file.getFileName.toString
name.endsWith(".pkg.tar.xz") || name.endsWith(".pkg.tar.zst")
}).foreach(file => {
if (!knownPkg.contains(file.getFileName.toString) || {
val query = em.createQuery("SELECT p.updTimestamp FROM PkgData p WHERE filename = :filename", classOf[Date])
query.setParameter("filename", file.getFileName.toString)
val result = query.getSingleResult
Files.getLastModifiedTime(file).toMillis > result.getTime
}) {
try {
parseFile(em, file)
} catch {
case e: Exception =>
logger.error(s"Error parsing '${file.toAbsolutePath}'", e)
if (em.getTransaction.getRollbackOnly) throw e
else Files.delete(file)
}
}
})
logger.info("Removing obsolete packages")
deleteOld(em)
logger.info("Repository cleanup completed successfully")
ctx.invalidateCache.set(true)
}
private def parseFile(em: EntityManager, file: Path) = {
val hquery = em.createQuery(hashQueryCount, classOf[java.lang.Long])
val data = Parser.parseFile(file, CompressionFormat.guess(file))
hquery.setParameter("md5sum", data.md5sum)
if (hquery.getSingleResult == 0) {
val fquery = em.createQuery("SELECT p FROM PkgData p WHERE p.fileName = :fileName", classOf[PkgData])
fquery.setParameter("fileName", file.getFileName.toString)
fquery.getResultList.forEach(deletePkgData(em, _))
em.persist(data)
logger.info(s"Persisting package ${file.getFileName}")
}
}
@TransactionAttribute(TransactionAttributeType.REQUIRED)
override def deletePackage(filename: String): Unit = {
val em = emf.createEntityManager
deletePackage(em, filename)
logger.info(s"Package $filename has been deleted")
}
private def deletePackage(em: EntityManager, filename: String): Unit = {
val fquery = em.createQuery("SELECT p FROM PkgData p WHERE fileName = :fileName", classOf[PkgData])
fquery.setParameter("fileName", filename)
val savedFiles = fquery.getResultList
if (savedFiles.size == 0) {
throw new RuntimeException(String.format("Package with name %s not found", filename))
}
val pkg = fquery.getResultList.get(0)
Files.delete(ctx.getFile(pkg))
em.remove(pkg)
}
final private val deleteQuery = "SELECT p.fileName FROM PkgData p WHERE p.buildDate < :cutoff and p.id.name in \n" + "(SELECT p2.id.name FROM PkgData p2 GROUP BY p2.id.name HAVING count(p2.id.name) > :minVersions\n)"
private def deleteOld(em: EntityManager): Unit = {
val query = em.createQuery(deleteQuery, classOf[String])
val cutoff = Calendar.getInstance
cutoff.add(Calendar.YEAR, -2)
query.setParameter("cutoff", cutoff.getTime)
query.setParameter("minVersions", 2.toLong)
val list = query.getResultList
list.forEach(deletePackage(_))
}
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
override def countResults(name: String, version: String, arch: String): Long = {
val em = emf.createEntityManager
QueryEngine.countResults(em, name, version, arch)
}
@TransactionAttribute(TransactionAttributeType.SUPPORTS)
override def searchPackage(name: String, version: String, arch: String, pageNumber: Int, pageSize: Int, fileName: String) = {
val em = emf.createEntityManager
QueryEngine.searchPackage(em, name, version, arch, pageNumber, pageSize, null)
}
}

448
src/main/scala/net/woggioni/jpacrepo/service/PacmanWebService.scala
View File

@@ -1,448 +0,0 @@
package net.woggioni.jpacrepo.service
import java.io._
import java.net.URI
import java.nio.file.{Files, Paths}
import java.nio.file.StandardCopyOption.ATOMIC_MOVE
import java.util
import javax.ejb._
import javax.inject.Inject
import javax.persistence._
import javax.ws.rs._
import javax.ws.rs.core._
import javax.xml.bind.annotation.{XmlElement, XmlRootElement}
import net.woggioni.jpacrepo.config.AppConfig
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.pacbase.{CompressionFormat, PkgData, PkgId, VersionOrdering}
import net.woggioni.jpacrepo.utils.Utils._
import net.woggioni.jpacrepo.version.{PkgIdComparator, VersionComparator}
import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
import org.slf4j.Logger
import scala.beans.BeanProperty
import scala.collection.SortedMap
import scala.collection.immutable.TreeMap
import scala.jdk.CollectionConverters._
@ApplicationPath("api")
class ApplicationConfig() extends Application {
val classes: Set[Class[_]] = Set(classOf[PacmanWebService])
override def getClasses = classes.asJava
}
object PacmanWebService {
val pkgIdOrdering: Ordering[PkgId] = Ordering.comparatorToOrdering(new PkgIdComparator)
val versionOrdering: Ordering[String] = Ordering.comparatorToOrdering(new VersionComparator)
}
@XmlRootElement
class PkgDataList extends util.ArrayList[PkgData] {
def this(l: util.List[PkgData]) {
this()
l.forEach(el => add(el))
}
def this(elements: PkgData*) {
this()
elements.foreach(el => add(el))
}
@XmlElement(name = "pkgData")
def getItems: util.List[PkgData] = this
def setItems(pkgs: util.List[PkgData]): Unit = {
this.clear()
this.addAll(pkgs)
}
}
@XmlRootElement
class StringList extends util.ArrayList[String] {
def this(l: util.List[String]) {
this()
l.forEach(el => add(el))
}
def this(elements: String*) {
this()
elements.foreach(el => add(el))
}
def this(elements: Iterable[String]) {
this()
elements.foreach(el => add(el))
}
@XmlElement(name = "string")
def getItems: util.List[String] = this
def setItems(pkgs: util.List[String]): Unit = {
this.clear()
this.addAll(pkgs)
}
}
@XmlRootElement
class PkgTuple {
@BeanProperty
var md5sum: String = _
@BeanProperty
var filename: String = _
@BeanProperty
var size: Long = _
}
@Singleton
@Path("/pkg")
@ConcurrencyManagement(ConcurrencyManagementType.BEAN)
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@Consumes(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
@TransactionManagement(TransactionManagementType.CONTAINER)
class PacmanWebService {
private var cachedMap: SortedMap[PkgId, PkgTuple] = _
@Inject
private var emf: EntityManagerFactory = _
@Inject
private var log: Logger = _
@Inject
private var service: PacmanServiceView = _
@Inject
private var ctx: AppConfig = _
private def getCachedMap: SortedMap[PkgId, PkgTuple] = {
var result: SortedMap[PkgId, PkgTuple] = null
if (!ctx.invalidateCache.get()) {
result = cachedMap
}
if (result == null) {
synchronized {
result = cachedMap
if (result == null) {
val em = emf.createEntityManager()
val query = em.createQuery(
"SELECT pkg.id.name, pkg.id.version, pkg.id.arch, pkg.fileName, pkg.size, pkg.md5sum " +
"FROM PkgData pkg ORDER BY pkg.id.name, pkg.id.version, pkg.id.arch",
classOf[Array[AnyRef]])
val stream = query.getResultList
.asScala
.to(LazyList)
.map(pkg => {
val name: String = pkg(0).asInstanceOf[String]
val version: String = pkg(1).asInstanceOf[String]
val arch: String = pkg(2).asInstanceOf[String]
val filename: String = pkg(3).asInstanceOf[String]
val size: Long = pkg(4).asInstanceOf[Long]
val md5sum: String = pkg(5).asInstanceOf[String]
val tuple: PkgTuple = new PkgTuple
tuple.filename = filename
tuple.size = size
tuple.md5sum = md5sum
val id = new PkgId()
id.name = name
id.version = version
id.arch = arch
id -> tuple
})
cachedMap = TreeMap.from(stream)(PacmanWebService.pkgIdOrdering)
ctx.invalidateCache.set(false)
result = cachedMap
}
}
}
result
}
@GET
@Path("searchByName/{name}")
def searchByName(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
if (name == null) throw new WebApplicationException(Response.Status.BAD_REQUEST)
val query: String = String.format("SELECT pkgId.name FROM PkgId pkgId WHERE LOWER(pkgId.name) LIKE '%%%s%%' ORDER BY pkgId.name", name)
Response.ok(em.createQuery(query, classOf[String]).getResultList).build
}
@GET
@Path("searchByHash/{md5sum}")
def searchByHash(@PathParam("md5sum") md5sum: String): Response = getPackageByHash(md5sum)
@GET
@Path("list/{name}")
def getPackage(@PathParam("name") name: String): Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT pkg.id.version FROM PkgData pkg WHERE pkg.id.name = :name ORDER BY pkg.id.version", classOf[String])
query.setParameter("name", name)
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("list/{name}/{version}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String): Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT pkg.arch FROM PkgData pkg WHERE pkg.id.name = :name AND pkg.id.version = :version ORDER BY pkg.id.arch", classOf[String])
query.setParameter("name", name)
query.setParameter("version", version)
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("list/{name}/{version}/{arch}")
def getPackage(@PathParam("name") name: String, @PathParam("version") version: String, @PathParam("arch") arch: String): Response = {
val em = emf.createEntityManager()
val query: TypedQuery[PkgData] = em.createQuery("SELECT pkg FROM PkgData pkg WHERE " + "pkg.id.name = :name AND " + "pkg.id.version = :version AND " + "pkg.id.arch = :arch " + "ORDER BY pkg.arch", classOf[PkgData])
query.setParameter("name", name)
query.setParameter("version", version)
query.setParameter("arch", arch)
Response.ok(query.getSingleResult).build
}
@GET
@Produces(Array(MediaType.APPLICATION_JSON))
@Path("map")
def getPackageMap(@Context request: Request): Response = {
val cc: CacheControl = new CacheControl
cc.setMaxAge(86400)
cc.setMustRevalidate(true)
cc.setNoCache(true)
val cachedMap = getCachedMap
val etag: EntityTag = new EntityTag(Integer.toString(cachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val result: util.Map[String, util.Map[String, util.Map[String, PkgTuple]]] = cachedMap.to(LazyList)
.groupBy(_._1.name).view.mapValues(t =>
SortedMap.from(t.groupBy(_._1.version).view.mapValues(
_.map(pair => pair._1.arch -> pair._2).to(SortedMap).asJava
))(PacmanWebService.versionOrdering.reverse).asJava
).toMap.asJava
builder = Response.ok(result)
builder.tag(etag)
}
builder.cacheControl(cc)
builder.build
}
@GET
@Path("hashes")
def getHashes: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.md5sum FROM PkgData p", classOf[String])
Response.ok(new StringList(query.getResultList)).build
}
@GET
@Path("files")
def getFiles: Response = {
val em = emf.createEntityManager()
val query = em.createQuery("SELECT p.fileName FROM PkgData p", classOf[String])
Response.ok(new StringList(query.getResultList)).build
}
private def getPackageByHash(md5sum: String): Response = {
val em = emf.createEntityManager()
val hquery = em.createNamedQuery("searchByHash", classOf[PkgData])
if (md5sum != null) hquery.setParameter("md5sum", md5sum)
manageQueryResult(hquery.getResultList, true)
}
private def getPackageByFileName(file: String): Response = {
val em = emf.createEntityManager()
val fnquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fnquery.setParameter("fileName", file)
manageQueryResult(fnquery.getResultList, true)
}
@GET
@Path("filesize/{filename}")
def getFileSize(@PathParam("filename") fileName: String, @Context request: Request): Response = {
val cc: CacheControl = new CacheControl
cc.setMaxAge(86400)
cc.setMustRevalidate(true)
cc.setNoCache(true)
val etag: EntityTag = new EntityTag(Integer.toString(getCachedMap.hashCode))
var builder: Response.ResponseBuilder = request.evaluatePreconditions(etag)
if (builder == null) {
val res = ctx.getFile(fileName)
if (!Files.exists(res)) throw new NotFoundException(String.format("File '%s' was not found", fileName))
builder = Response.ok(Files.size(res))
builder.tag(etag)
}
builder.cacheControl(cc)
builder.build
}
@GET
@Path("download/{filename}")
@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
def downloadPackage(@PathParam("filename") fileName: String): Response = {
val em = emf.createEntityManager()
val fnquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fnquery.setParameter("fileName", fileName)
try {
val pkg: PkgData = fnquery.getSingleResult
val stream: StreamingOutput = (output: OutputStream) => {
Files.newInputStream(ctx.getFile(pkg)).use { is =>
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ( {
read = is.read(bytes)
read >= 0
}) {
output.write(bytes, 0, read)
}
}
()
}
Response.ok(stream).header("Content-Length", Files.size(ctx.getFile(pkg))).build
} catch {
case _: NoResultException =>
throw new NotFoundException
}
}
@POST
@Path("/doYouWantAny")
@Produces(Array(MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON))
def doYouWantAny(filenames: util.List[String]): Response = {
val em = emf.createEntityManager()
val result = Set.from(filenames.asScala)
if (result.nonEmpty) {
val query = em.createQuery("SELECT pkg.fileName from PkgData pkg WHERE pkg.fileName in :filenames", classOf[String])
query.setParameter("filenames", filenames)
val toBeRemoved = Set.from(query.getResultList.asScala)
Response.ok(new StringList(result -- toBeRemoved)).build
}
else {
Response.ok(result.toArray).build
}
}
@POST
@Path("/upload")
@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Consumes(Array("application/x-xz", "application/gzip", "application/x-tar", MediaType.APPLICATION_OCTET_STREAM))
def createPackage(input: InputStream,
@MatrixParam("filename") filename: String,
@Context uriInfo: UriInfo): Response = {
val em = emf.createEntityManager()
if (filename == null) throw new BadRequestException
val file = Files.createTempFile(ctx.repoFolder, filename, null)
val fquery: TypedQuery[PkgData] = em.createNamedQuery("searchByFileName", classOf[PkgData])
fquery.setParameter("fileName", filename)
val savedFiles: util.List[PkgData] = fquery.getResultList
if (savedFiles.size > 0) Response.notModified.build
else {
Files.newOutputStream(file).useCatch { os =>
val buffer: Array[Byte] = new Array[Byte](0x1000)
var read = 0
while ( {
read = input.read(buffer)
read >= 0
}) {
os.write(buffer, 0, read)
}
val pkg = Parser.parseFile(file, CompressionFormat.guess(Paths.get(filename)))
pkg.fileName = filename
Option(em.find(classOf[PkgData], pkg.id)) match {
case Some(pkgData) => {
em.remove(pkgData)
Files.delete(ctx.repoFolder.resolve(pkgData.fileName))
}
case None =>
}
log.info(s"Persisting package ${pkg.fileName}")
em.persist(pkg)
val pkgUri: URI = uriInfo.getAbsolutePathBuilder.path(pkg.fileName).build()
Files.move(file, ctx.repoFolder.resolve(filename), ATOMIC_MOVE)
ctx.invalidateCache.set(true)
cachedMap = null
Response.created(pkgUri).build
} { t: Throwable =>
Files.delete(file)
throw t
}
}
}
@GET
@Path("/search")
def searchPackage(
@QueryParam("name") name: String,
@QueryParam("version") version: String,
@QueryParam("arch") arch: String,
@QueryParam("page") pageNumber: Int,
@QueryParam("pageSize") pageSize: Int,
@QueryParam("fileName") fileName: String): util.List[PkgData] = {
service.searchPackage(name, version, arch, pageNumber, pageSize, fileName)
}
@OPTIONS
@Path("/downloadTar")
@Produces(Array("text/plain; charset=UTF-8"))
def options: Response = Response.ok("POST, OPTIONS").build
@POST
@Path("/downloadTar")
@Produces(Array("application/x-tar"))
@Consumes(Array(MediaType.APPLICATION_FORM_URLENCODED))
def downloadTar(@FormParam("pkgs") formData: String): Response = {
val files: Array[String] = formData.split(" ")
files.find(fileName => !Files.exists(ctx.getFile(fileName))) match {
case Some(fileName) => throw new NotFoundException(s"Package file '$fileName' does not exist")
case None =>
}
val stream = new StreamingOutput() {
override def write(output: OutputStream): Unit = {
val taos: TarArchiveOutputStream = new TarArchiveOutputStream(output)
try {
for (fname <- files) {
val file = ctx.getFile(fname)
Files.newInputStream(file).use { input =>
val entry: TarArchiveEntry = new TarArchiveEntry(fname)
entry.setSize(Files.size(file))
taos.putArchiveEntry(entry)
val bytes: Array[Byte] = new Array[Byte](1024)
var read = 0
while ( {
read = input.read(bytes)
read >= 0
}) {
taos.write(bytes, 0, read)
}
taos.closeArchiveEntry()
}
}
} finally {
taos.close()
}
}
}
Response.ok(stream).header("Content-Disposition", "attachment; filename=pkgs.tar").build
}
private def manageQueryResult(list: util.List[PkgData]): Response = manageQueryResult(list, false)
private def manageQueryResult(list: util.List[PkgData], singleResult: Boolean): Response = {
if (list.isEmpty) throw new NotFoundException
else if (singleResult) {
if (list.size == 1) Response.ok(list.get(0)).build
else throw new NonUniqueResultException("The returned list does not contain a single element")
} else {
Response.ok(new PkgDataList(list)).build
}
}
}

32
src/main/scala/net/woggioni/jpacrepo/utils/Utils.scala
View File

@@ -1,32 +0,0 @@
package net.woggioni.jpacrepo.utils
object Utils {
implicit class Use[CLOSEABLE <: AutoCloseable, RESULT](closeable: CLOSEABLE) {
def useCatch(cb: CLOSEABLE => RESULT) (exceptionally : Throwable => RESULT): RESULT = {
try {
cb(closeable)
} catch {
case t : Throwable => exceptionally(t)
} finally {
closeable.close()
}
}
def use(cb: CLOSEABLE => RESULT): RESULT = {
try {
cb(closeable)
} finally {
closeable.close()
}
}
}
implicit class ContextFunctions[IN, OUT](in: IN) {
def let(cb: (IN) => OUT): OUT = cb(in)
def also(cb: (IN) => Unit) : IN = {
cb(in)
in
}
}
}

105
src/test/java/ClientTest.java
View File

@@ -1,30 +1,39 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.model.Hasher;
import net.woggioni.jpacrepo.model.MD5InputStream;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.pacbase.CompressionFormat;
import net.woggioni.jpacrepo.pacbase.PkgData;
import net.woggioni.jpacrepo.service.PacmanServiceRemote;
import net.woggioni.jpacrepo.api.model.PkgData;
import net.woggioni.jpacrepo.api.service.PacmanServiceRemote;
import net.woggioni.jpacrepo.impl.model.CompressionFormatImpl;
import net.woggioni.jpacrepo.impl.model.PkgDataImpl;
import net.woggioni.jwo.Con;
import net.woggioni.jwo.Fun;
import net.woggioni.jwo.Hash;
import net.woggioni.jwo.JWO;
import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
import org.jboss.resteasy.spi.ResteasyProviderFactory;
import javax.naming.*;
import javax.ws.rs.client.*;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import jakarta.ws.rs.client.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.UriBuilder;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.Properties;
import java.util.stream.Stream;
public class ClientTest {
@@ -59,8 +68,7 @@ public class ClientTest {
}
}
public void testPUT() throws Exception
{
public void testPUT() throws Exception {
ResteasyProviderFactory instance = ResteasyProviderFactory.getInstance();
RegisterBuiltin.register(instance);
Client client = ClientBuilder.newClient();
@@ -77,60 +85,39 @@ public class ClientTest {
assert Response.Status.CREATED.getStatusCode() == response.getStatus();
}
public void hashTest() throws Exception
{
String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
for (String file : files)
{
MessageDigest md = MessageDigest.getInstance("MD5");
try (InputStream is = Files.newInputStream(Paths.get(file)))
{
DigestInputStream dis = new DigestInputStream(is, md);
dis.on(true);
// is.read();
long a = new File(file).length();
byte[] out = new byte[(int)a];
dis.read(out, 0, (int) a);
@Test
public void hashTest(@TempDir Path testDir) {
ClassLoader cl = getClass().getClassLoader();
Stream.of("gvfs-nfs-1.50.2-1-x86_64.pkg.tar.zst")
.forEach((Con<String>) resourceName -> {
Path tmpFile = testDir.resolve(resourceName);
try(InputStream is = cl.getResourceAsStream(resourceName)) {
Files.copy(is, tmpFile);
}
System.out.println(Hasher.bytesToHex(md.digest()));
System.out.println(Hasher.computeMD5(new FileInputStream(file)));
InputStream fis = new FileInputStream(file);
MD5InputStream h = new MD5InputStream(fis);
long a = new File(file).length();
byte[] out = new byte[(int)a];
h.read(out, 0, (int) a);
System.out.println(h.digest());
Path path = Paths.get(file);
PkgData p = Parser.parseFile(path, CompressionFormat.guess(path));
System.out.println(p.md5sum());
}
Hash hash;
try(InputStream is = Files.newInputStream(tmpFile)) {
hash = Hash.md5(is);
}
PkgData p = PkgDataImpl.parseFile(tmpFile, CompressionFormatImpl.guess(tmpFile));
Assertions.assertEquals(JWO.bytesToHex(hash.getBytes()), p.getMd5sum());
});
}
private static void traverseJndiNode(String nodeName, Context context)
{
try
{
private static void traverseJndiNode(String nodeName, Context context) {
try {
NamingEnumeration<NameClassPair> list = context.list(nodeName);
while (list.hasMore())
{
while (list.hasMore()) {
String childName = nodeName + "" + list.next().getName();
System.out.println(childName);
traverseJndiNode(childName, context);
}
} catch (NamingException ex)
{
} catch (NamingException ex) {
// We reached a leaf
}
}
public void invokeStatelessBean() throws Exception
{
@Test
public void invokeStatelessBean() throws Exception {
Properties prop = new Properties();
InputStream in = getClass().getClassLoader().getResourceAsStream("jboss-ejb-client.properties");
prop.load(in);
@@ -140,10 +127,10 @@ public class ClientTest {
prop.put(Context.PROVIDER_URL, "http-remoting://localhost:8080");
// prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
// prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
prop.put(Context.SECURITY_PRINCIPAL, "walter");
prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d");
// prop.put(Context.SECURITY_PRINCIPAL, "admin");
// prop.put(Context.SECURITY_CREDENTIALS, "123456");
// prop.put(Context.SECURITY_PRINCIPAL, "walter");
// prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d");
prop.put(Context.SECURITY_PRINCIPAL, "luser");
prop.put(Context.SECURITY_CREDENTIALS, "123456");
prop.put("jboss.naming.client.ejb.context", true);
Context context = new InitialContext(prop);
@@ -151,7 +138,7 @@ public class ClientTest {
traverseJndiNode("/", context);
// final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
final PacmanServiceRemote service = (PacmanServiceRemote) ctx.lookup(
"/jpacrepo_2.12-2.0/PacmanServiceEJB!net.woggioni.jpacrepo.service.PacmanServiceRemote"
"/jpacrepo-2.0-SNAPSHOT/PacmanServiceEJB!net.woggioni.jpacrepo.service.PacmanServiceRemote"
);
// List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
// System.out.println(new XStream().toXML(pkgs));

6
src/test/java/ParseTest.java
View File

@@ -1,9 +1,9 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
import lombok.SneakyThrows;
import net.woggioni.jpacrepo.model.Parser;
import net.woggioni.jpacrepo.api.model.PkgData;
import net.woggioni.jpacrepo.impl.model.PkgDataImpl;
import net.woggioni.jpacrepo.pacbase.CompressionFormat;
import net.woggioni.jpacrepo.pacbase.PkgData;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
@@ -26,7 +26,7 @@ public class ParseTest {
Files.list(Paths.get("/var/cache/pacman/pkg"))
.filter(Files::isRegularFile)
.filter(p -> pattern.matcher(p.getFileName().toString()).matches())
.map(path -> Parser.parseFile(path, CompressionFormat.guess(path)))
.map(path -> PkgDataImpl.parseFile(path, CompressionFormat.guess(path)))
.limit(10)
.map(new Function<PkgData, String>() {
@Override

4
src/test/java/net/woggioni/jpacrepo/annotation/UnitTesting.java
View File

@@ -1,7 +1,7 @@
package net.woggioni.jpacrepo.annotation;
import javax.enterprise.inject.Alternative;
import javax.enterprise.inject.Stereotype;
import jakarta.enterprise.inject.Alternative;
import jakarta.enterprise.inject.Stereotype;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;

BIN
src/test/resources/gvfs-nfs-1.50.2-1-x86_64.pkg.tar.zst Normal file
View File

Binary file not shown.

3
src/test/resources/log4j2.xml
View File

@@ -1,8 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
<Properties>
<Property name="log-path">$${env:APOLLO_ENVIRONMENT_ROOT:-build}/var/logs</Property>
</Properties>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss,SSS} %highlight{[%p]} (%t) %c: %m%n"/>

49
src/test/scala/net/woggioni/jpacrepo/client/SyncDbTest.scala
View File

@@ -1,49 +0,0 @@
package net.woggioni.jpacrepo.client
import java.io.InputStream
import java.util.Properties
import javax.naming.{Context, InitialContext, NameClassPair, NamingEnumeration, NamingException}
import net.woggioni.jpacrepo.service.PacmanServiceRemote
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
class SyncDbTest extends AnyFlatSpec {
private def traverseJndiNode(nodeName: String, context: Context) {
try {
val list = context.list(nodeName)
while (list.hasMore) {
val childName = nodeName + list.next.getName
System.out.println(childName)
traverseJndiNode(childName, context)
}
} catch {
case _ : NamingException =>
}
}
"it" should "be possible to invoke syncDB remotely" in {
val prop = new Properties
val in = getClass.getClassLoader.getResourceAsStream("jboss-ejb-client.properties")
prop.load(in)
prop.put(Context.URL_PKG_PREFIXES, "org.jboss.ejb.client.naming")
prop.put(Context.INITIAL_CONTEXT_FACTORY, "org.jboss.naming.remote.client.InitialContextFactory")
prop.put(Context.PROVIDER_URL, "http-remoting://localhost:5080")
// prop.put(Context.PROVIDER_URL, "http-remoting://nuc:8080");
// prop.put(Context.PROVIDER_URL, "remote://odroid-u3:4447");
prop.put(Context.SECURITY_PRINCIPAL, "walter")
prop.put(Context.SECURITY_CREDENTIALS, "27ff5990757d1d")
// prop.put(Context.SECURITY_PRINCIPAL, "admin");
// prop.put(Context.SECURITY_CREDENTIALS, "123456");
prop.put("jboss.naming.client.ejb.context", true)
val context = new InitialContext(prop)
val ctx = new InitialContext(prop)
traverseJndiNode("/", context)
// final PacmanService stateService = (PacmanService) ctx.lookup("/jpacrepo-1.0/remote/PacmanServiceEJB!service.PacmanService");
val service = ctx.lookup("/jpacrepo_2.13-2.0/PacmanServiceEJB!net.woggioni.jpacrepo.service.PacmanServiceRemote").asInstanceOf[PacmanServiceRemote]
// List<PkgData> pkgs = service.searchPackage("google-earth", null, null, 1, 10);
// System.out.println(new XStream().toXML(pkgs));
service.syncDB()
}
}

26
src/test/scala/net/woggioni/jpacrepo/config/AppConfigTest.scala
View File

@@ -1,26 +0,0 @@
package net.woggioni.jpacrepo.config
import net.woggioni.jpacrepo.factory.BeanFactory
import org.jboss.weld.environment.se.Weld
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.must.Matchers
class AppConfigTest extends AnyFlatSpec with Matchers {
private val weld = new Weld
weld.disableDiscovery()
// weld.alternatives(classOf[TestPersistenceProducer])
weld.beanClasses(
// classOf[PacmanServiceEJB],
// classOf[PacmanWebService],
// classOf[AppConfig],
classOf[BeanFactory],
)
val container = weld.initialize()
"test" should "pass" in {
val appConfig = container.select(classOf[AppConfig]).get()
println(appConfig.repoFolder)
}
}

49
src/test/scala/net/woggioni/jpacrepo/pacbase/MarshalTest.scala
View File

@@ -1,49 +0,0 @@
package net.woggioni.jpacrepo.pacbase
import java.nio.file.{Files, Path, Paths}
import java.util
import java.util.regex.Pattern
import java.util.stream.Collectors
import javax.xml.bind.annotation.{XmlAccessType, XmlAccessorType, XmlElement, XmlRootElement}
import javax.xml.bind.{JAXBContext, Marshaller}
import net.woggioni.jpacrepo.model.Parser
import net.woggioni.jpacrepo.service.PkgDataList
import org.scalatest.flatspec.AnyFlatSpec
import scala.jdk.CollectionConverters._
import scala.annotation.meta.field
@XmlRootElement(name = "List")
@XmlAccessorType(XmlAccessType.FIELD)
class JaxbList2[T] private (@(XmlElement @field)(name = "Item") private val list : util.List[T]) {
def this(s : Seq[T]) {
this(s.toList.asJava)
}
def this() {
this(new util.ArrayList[T]())
}
}
class MarshalTest extends AnyFlatSpec {
"asdfs" should "sdfsd" in {
val context = JAXBContext.newInstance(classOf[PkgId], classOf[PkgDataList])
val mar = context.createMarshaller
mar.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true)
val pkgId2 = new PkgId
pkgId2.name = "linux"
pkgId2.version = "4.6.1"
pkgId2.arch = "x86_64"
val pattern = Pattern.compile(".*\\.pkg\\.tar\\.(zst)")
val pkgDatas = Files.list(Paths.get("/var/cache/pacman/pkg"))
.filter(Files.isRegularFile(_))
.filter((p: Path) => pattern.matcher(p.getFileName.toString).matches)
.limit(10)
.map(Parser.parseFile(_, CompressionFormat.Z_STANDARD))
.collect(Collectors.toList[PkgData])
val list = new PkgDataList(pkgDatas)
mar.marshal(list, System.out)
}
}

11
src/test/scala/net/woggioni/jpacrepo/persistence/TestPersistenceProducer.scala
View File

@@ -1,11 +0,0 @@
package net.woggioni.jpacrepo.persistence
import javax.enterprise.inject.Produces
import javax.persistence.Persistence
import net.woggioni.jpacrepo.annotation.UnitTesting
class TestPersistenceProducer {
@Produces
@UnitTesting
def createEntityManagerFactory = Persistence.createEntityManagerFactory("test")
}

38
src/test/scala/net/woggioni/jpacrepo/sample/ExampleSpec.scala
View File

@@ -1,38 +0,0 @@
package net.woggioni.jpacrepo.sample
import net.woggioni.jpacrepo.persistence.InitialSchemaAction
import net.woggioni.jpacrepo.version.VersionComparator
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import scala.collection.mutable
class ExampleSpec extends AnyFlatSpec with Matchers {
def foo(a : Int)(b : String = "walter"): Unit = {
println(s"a: $a, b: $b")
}
def foo(c : Int): Unit = {
foo(a=c)("adfsda")
}
// "A Stack" should "pop values in last-in-first-out order" in {
// val stack = new mutable.Stack[Int]
// stack.push(1)
// stack.push(2)
// stack.pop() should be (2)
// stack.pop() should be (1)
// }
//
// it should "throw NoSuchElementException if an empty stack is popped" in {
// val emptyStack = new mutable.Stack[Int]
// a [NoSuchElementException] should be thrownBy {
// emptyStack.pop()
// }
// }
"sdfgf" should "dfgfd" in {
foo(a=5)()
}
}

58
src/test/scala/net/woggioni/jpacrepo/service/PacmanServiceEJBTest.scala
View File

@@ -1,58 +0,0 @@
package net.woggioni.jpacrepo.service
import javax.enterprise.util.TypeLiteral
import net.woggioni.jpacrepo.factory.BeanFactory
import net.woggioni.jpacrepo.persistence.TestPersistenceProducer
import org.jboss.weld.environment.se.Weld
//object WeldContainer {
// private val weld = new Weld
// weld.disableDiscovery()
//
// weld.beanClasses(
// classOf[TestPersistenceProducer],
// classOf[PacmanServiceEJB],
// classOf[ApplicationContext],
// classOf[BeanFactory],
// )
//
// private val count = new AtomicInteger(0)
//}
//class WeldContainer extends AutoCloseable {
//
// val container = WeldContainer.weld.initialize()
// WeldContainer.count.incrementAndGet()
//
// override def close(): Unit = {
// container.close()
// if(WeldContainer.count.decrementAndGet() == 0) {
//// WeldContainer.weld.shutdown()
// }
// }
//}
class PacmanServiceEJBTest {
private val weld = new Weld
// weld.disableDiscovery()
weld.alternatives(classOf[TestPersistenceProducer])
////
weld.beanClasses(
classOf[PacmanServiceEJB],
classOf[PacmanWebService],
// classOf[ApplicationContext],
classOf[BeanFactory],
)
def test = {
val container = weld.initialize()
try {
val s = getClass.getResourceAsStream("/log4j2.xml")
val service = container.select(new TypeLiteral[PacmanServiceView] {}).get()
service.syncDB()
} finally {
container.close()
}
}
}

33
src/test/scala/net/woggioni/jpacrepo/service/PacmanWebServiceTest.scala
View File

@@ -1,33 +0,0 @@
package net.woggioni.jpacrepo.service
class PacmanWebServiceTest {
// val server = {
// val server = new UndertowJaxrsServer()
// server.start()
// import org.jboss.resteasy.spi.ResteasyDeployment
// val deployment = new ResteasyDeployment
// deployment.setInjectorFactoryClass("org.jboss.resteasy.cdi.CdiInjectorFactory")
// deployment.setApplicationClass(classOf[ApplicationConfig].getName)
// val di: DeploymentInfo = server.undertowDeployment(deployment)
// di.setClassLoader(classOf[ApplicationConfig].getClassLoader)
// di.setContextPath("/jpacrepo")
// di.setDeploymentName("jpacrepo")
// di.addListeners(Servlets.listener(classOf[Listener]))
// server.deploy(di)
// server
// }
// @Test
// def foo {
// val client = ClientBuilder.newClient()
// val webTarget = client.target(TestPortProvider.generateURL("/jpacrepo/rest/pkg/map"))
// val response = webTarget.request().get()
// val res = response.getEntity
// response.getStatus
// }
def boh = {
println(System.getProperty("net.woggioni.jpacrepo.configuration.file"))
}
}

16
src/test/scala/net/woggioni/jpacrepo/version/VersionComparatorSpec.scala
View File

@@ -1,16 +0,0 @@
package net.woggioni.jpacrepo.version
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import scala.util.Random
class VersionComparatorSpec extends AnyFlatSpec with Matchers {
"Version sorting" should "work as expected" in {
val vc = new VersionComparator
val originalList = List("1.0", "2.0", "3.0", "5.6.7.arch1-1", "5.6.7.arch3-1", "5.6.7.arch3-2", "20200421.78c0348-1")
val shuffledList = new Random(101325).shuffle(originalList)
val sortedList = shuffledList.sorted(Ordering.comparatorToOrdering(vc))
sortedList should be (originalList)
}
}