From 9b377885289c599dd64010f349f052d925564ad1 Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Thu, 26 Mar 2015 07:26:19 +0100 Subject: [PATCH] initial commit --- build.gradle | 23 ++ jpacrepo.iml | 29 ++ settings.gradle | 2 + src/main/java/model/PkgData.java | 79 ++++++ src/main/java/model/PkgName.java | 24 ++ src/main/java/pacbase/Parser.java | 177 ++++++++++++ src/main/java/service/ApplicationConfig.java | 39 +++ src/main/java/service/PacmanService.java | 273 +++++++++++++++++++ src/main/resources/META-INF/persistence.xml | 17 ++ src/test/java/ParseTest.java | 38 +++ 10 files changed, 701 insertions(+) create mode 100644 build.gradle create mode 100644 jpacrepo.iml create mode 100644 settings.gradle create mode 100644 src/main/java/model/PkgData.java create mode 100644 src/main/java/model/PkgName.java create mode 100644 src/main/java/pacbase/Parser.java create mode 100644 src/main/java/service/ApplicationConfig.java create mode 100644 src/main/java/service/PacmanService.java create mode 100644 src/main/resources/META-INF/persistence.xml create mode 100644 src/test/java/ParseTest.java diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..dabe5ea --- /dev/null +++ b/build.gradle @@ -0,0 +1,23 @@ +apply plugin: 'java' +apply plugin: 'war' + +sourceCompatibility = 1.8 +version = '1.0' + +repositories { + mavenCentral() + mavenLocal() +} + +dependencies { + testCompile group: 'junit', name: 'junit', version: '4.11' + compile 'org.apache.commons:commons-compress:1.9' + compile 'org.tukaani:xz:1.5' + compile 'javax:javaee-api:7.0' + compile 'commons-io:commons-io:2.4' + testCompile 'com.thoughtworks.xstream:xstream:1.4.8' +} + +task deployWildfly(dependsOn: 'war') << { + '/opt/wildfly/bin/jboss-cli.sh --connect --user=admin --password=qwerty --command="deploy build/libs/jpacrepo-1.0.war --force"'.execute().waitFor() +} \ No newline at end of file diff --git a/jpacrepo.iml b/jpacrepo.iml new file mode 100644 index 0000000..1e811ae --- /dev/null +++ b/jpacrepo.iml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..7c2e743 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'jpacrepo' + diff --git a/src/main/java/model/PkgData.java b/src/main/java/model/PkgData.java new file mode 100644 index 0000000..478754c --- /dev/null +++ b/src/main/java/model/PkgData.java @@ -0,0 +1,79 @@ +package model; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Date; +import java.util.List; + +/** + * Created by walter on 22/03/15. + */ + + +@Entity +@XmlRootElement +@NamedQuery(name="searchById", query = "SELECT p FROM PkgData p WHERE p.id = :id") +public class PkgData +{ + @Id + @GeneratedValue + Integer id; + + @ManyToOne(cascade = CascadeType.PERSIST) + public PkgName name; + + public String base; + + public String version; + + public String description; + + public String url; + + @Temporal(TemporalType.TIMESTAMP) + public Date buildDate; + + public String packager; + + public Long size; + + public String arch; + + public String license; + + public String md5sum; + + public String filePath; + + public String fileName; + + @ElementCollection(fetch = FetchType.EAGER) + public List replaces; + + @ElementCollection(fetch = FetchType.EAGER) + public List conflict; + + @ElementCollection(fetch = FetchType.EAGER) + public List provides; + + @ElementCollection(fetch = FetchType.EAGER) + public List depend; + + @ElementCollection(fetch = FetchType.EAGER) + public List optdepend; + + @ElementCollection(fetch = FetchType.EAGER) + public List makedepend; + + @ElementCollection(fetch = FetchType.EAGER) + public List makeopkgopt; + + @ElementCollection(fetch = FetchType.EAGER) + public List backup; + + public String getFileName() + { + return String.format("%s-%s-%s.pkg.tar.xz", name, version, arch); + } + +} diff --git a/src/main/java/model/PkgName.java b/src/main/java/model/PkgName.java new file mode 100644 index 0000000..9880339 --- /dev/null +++ b/src/main/java/model/PkgName.java @@ -0,0 +1,24 @@ +package model; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Created by walter on 22/03/15. + */ + +@Entity +@XmlRootElement +public class PkgName +{ + @Id + public String id; + + public PkgName(){}; + + public PkgName(String name) + { + id = name; + } +} diff --git a/src/main/java/pacbase/Parser.java b/src/main/java/pacbase/Parser.java new file mode 100644 index 0000000..f6f5e30 --- /dev/null +++ b/src/main/java/pacbase/Parser.java @@ -0,0 +1,177 @@ +package pacbase; + +import model.PkgData; +import model.PkgName; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; + +/** + * Created by walter on 22/03/15. + */ +public class Parser +{ + static private final MessageDigest md5; + + static + { + try + { + md5 = MessageDigest.getInstance("MD5"); + } + catch (NoSuchAlgorithmException e) + { + throw new RuntimeException(e); + } + } + + public static PkgData parseFile(File file) throws Exception + { + FileInputStream fis = new FileInputStream(file); + XZCompressorInputStream xzcis = new XZCompressorInputStream(fis); + TarArchiveInputStream tais = new TarArchiveInputStream(xzcis); + ArchiveEntry ae; + while((ae = tais.getNextEntry()) != null) + { + if(ae.getName().equals(".PKGINFO")) + { + Map> propMap = new HashMap<>(); + byte[] buffer = new byte[tais.getRecordSize()]; + tais.read(buffer); + String info = new String(buffer, Charset.forName("UTF8")); + for(String line : info.split("\n")) + { + if(line.startsWith("#")) + { + continue; + } + else + { + String[] pair = line.split(" = "); + if(pair.length > 2) + { + throw new RuntimeException("Error parsing .PKGINFO file"); + } + else if(pair.length == 2) + { + String key = pair[0].trim(); + if(propMap.get(key) == null) + { + propMap.put(key, new ArrayList<>()); + } + propMap.get(key).add(pair[1].trim()); + } + } + } + tais.close(); + xzcis.close(); + fis.close(); + PkgData data = new PkgData(); + + for(String key : propMap.keySet()) + { + switch(key) + { + case "size": + data.size = Long.valueOf(propMap.get(key).get(0)); + break; + case "arch": + data.arch = propMap.get(key).get(0); + break; + case "replaces": + data.replaces = propMap.get(key); + break; + case "packager": + data.packager = propMap.get(key).get(0); + break; + case "url": + data.url = propMap.get(key).get(0); + break; + case "pkgname": + data.name = new PkgName(propMap.get(key).get(0)); + break; + case "builddate": + data.buildDate = new Date(Long.valueOf(propMap.get(key).get(0))*1000); + break; + case "license": + data.license = propMap.get(key).get(0); + break; + case "pkgver": + data.version = propMap.get(key).get(0); + break; + case "pkgdesc": + data.description = propMap.get(key).get(0); + break; + case "provides": + data.provides = propMap.get(key); + break; + case "conflict": + data.conflict = propMap.get(key); + break; + case "backup": + data.backup = propMap.get(key); + break; + case "optdepend": + data.optdepend = propMap.get(key); + break; + case "depend": + data.depend = propMap.get(key); + break; + case "makedepend": + data.makedepend = propMap.get(key); + break; + case "makepkgopt": + data.makeopkgopt = propMap.get(key); + break; + case "pkgbase": + data.base = propMap.get(key).get(0); + break; + + } + } + data.md5sum = computeMD5(new FileInputStream(file)); + data.filePath = file.getAbsolutePath(); + data.fileName = file.getName(); + return data; + } + } + tais.close(); + xzcis.close(); + fis.close(); + return null; + } + + public static String computeMD5(InputStream is) throws IOException + { + md5.reset(); + while(is.available()>0) + { + byte[] buffer2 = new byte[1000000]; + is.read(buffer2, 0, 1000000); + md5.update(buffer2); + } + is.close(); + return bytesToHex(md5.digest()); + } + + final protected static char[] hexArray = "0123456789ABCDEF".toCharArray(); + + public static String bytesToHex(byte[] bytes) { + char[] hexChars = new char[bytes.length * 2]; + for ( int j = 0; j < bytes.length; j++ ) { + int v = bytes[j] & 0xFF; + hexChars[j * 2] = hexArray[v >>> 4]; + hexChars[j * 2 + 1] = hexArray[v & 0x0F]; + } + return new String(hexChars); + } +} diff --git a/src/main/java/service/ApplicationConfig.java b/src/main/java/service/ApplicationConfig.java new file mode 100644 index 0000000..94ec6b7 --- /dev/null +++ b/src/main/java/service/ApplicationConfig.java @@ -0,0 +1,39 @@ +package service; + +import javax.ws.rs.ApplicationPath; +import javax.ws.rs.core.Application; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@ApplicationPath("rs") +public class ApplicationConfig extends Application +{ + + // ====================================== + // = Attributes = + // ====================================== + + private final Set> classes; + + // ====================================== + // = Constructors = + // ====================================== + + public ApplicationConfig() { + HashSet> c = new HashSet<>(); + c.add(PacmanService.class); + + classes = Collections.unmodifiableSet(c); + } + + // ====================================== + // = Getters & Setters = + // ====================================== + + @Override + public Set> getClasses() { + return classes; + } + +} diff --git a/src/main/java/service/PacmanService.java b/src/main/java/service/PacmanService.java new file mode 100644 index 0000000..e872973 --- /dev/null +++ b/src/main/java/service/PacmanService.java @@ -0,0 +1,273 @@ +package service; + +import model.PkgData; +import model.PkgName; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.RegexFileFilter; +import pacbase.Parser; + +import javax.annotation.PostConstruct; +import javax.ejb.Singleton; +import javax.ejb.Startup; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.TypedQuery; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.io.*; +import java.net.URI; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Startup +@Path("/pkg") +@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) +@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) +@Singleton +public class PacmanService +{ + private Properties prop; + + @PersistenceContext(unitName = "jpacrepo_pu") + private EntityManager em; + + + @Context + private UriInfo uriInfo; + + private Logger log = Logger.getLogger(PacmanService.class.getName()); + + + private String nameQuery = "SELECT pname FROM PkgName pname WHERE id = :name"; + private String fileQuery = "SELECT pdata FROM PkgData pdata WHERE name.id = :name AND version = :version AND arch = :arch"; + private String fileNameQuery = "SELECT pdata FROM PkgData pdata WHERE fileName = :fileName"; + private String hashQuery = "SELECT pdata FROM PkgData pdata WHERE md5sum = :md5sum"; + private String idQuery = "SELECT pdata FROM PkgData pdata WHERE id = :id"; + + @PostConstruct + public void syncDB() + { + loadProperties(); + + //Elimina i pacchetti sul DB che non esistono più nel filesystem + List listaDB = em.createQuery("SELECT p FROM PkgData p", PkgData.class).getResultList(); + for (PkgData p : listaDB) + { + File file = new File(p.filePath); + if (!file.exists()) + { + log.log(Level.INFO, String.format("Removing package %s", file.getName())); + em.remove(p); + } + } + + //Aggiunge sul DB i pacchetti presenti nel filesystem + Collection ls = FileUtils.listFiles(new File(prop.getProperty("RepoFolder")), new RegexFileFilter(".*\\.pkg\\.tar\\.xz"), DirectoryFileFilter.DIRECTORY); + int i = 0; + + TypedQuery fquery = em.createQuery(fileQuery, PkgData.class); + TypedQuery nquery = em.createQuery(nameQuery, PkgName.class); + + for (File file : ls) + { + try + { + PkgData data = Parser.parseFile(file); + + fquery.setParameter("name", data.name.id); + fquery.setParameter("version", data.version); + fquery.setParameter("arch", data.arch); + + List savedFiles = fquery.getResultList(); + + if (savedFiles.size() > 0) continue; + + nquery.setParameter("name", data.name.id); + List savedName = nquery.getResultList(); + if (savedName.size() > 0) + { + data.name = savedName.get(0); + } + em.persist(data); + log.log(Level.INFO, String.format("Persisting package %s", file.getName())); + } catch (Exception e) + { + throw new RuntimeException(e); + } + } + + } + + private void loadProperties() + { + prop = new Properties(); + InputStream input = null; + + try + { + input = new FileInputStream("/etc/jpacrepo/config.properties"); + // load a properties file + prop.load(input); + // get the property value and print it out + } catch (IOException ex) + { + throw new RuntimeException(ex); + } finally + { + if (input != null) + { + try + { + input.close(); + } catch (IOException e) + { + throw new RuntimeException(e); + } + } + } + + } + + /** + * JSON : curl -X GET -H "Accept: application/json" http://localhost:8080/chapter15-service-1.0/rs/book/1 -v + * XML : curl -X GET -H "Accept: application/xml" http://localhost:8080/chapter15-service-1.0/rs/book/1 -v + */ + @GET + @Path("search") + public Response getPackage(@QueryParam("name") String name, @QueryParam("version") String version, @QueryParam("arch") String arch) + { + TypedQuery fquery = em.createQuery(fileQuery, PkgData.class); + if(name != null) fquery.setParameter("name", name); + if(version != null) fquery.setParameter("version", version); + if(arch != null) fquery.setParameter("arch", arch); + + PkgData pkg = fquery.getSingleResult(); + if (pkg == null) + { + throw new NotFoundException(); + } + return Response.ok(pkg).build(); + } + + @GET + @Path("searchHash") + public Response getPackageByHash(@QueryParam("md5") String md5sum) + { + TypedQuery hquery = em.createQuery(hashQuery, PkgData.class); + if(md5sum != null) hquery.setParameter("md5sum", md5sum); + + PkgData pkg = hquery.getSingleResult(); + if (pkg == null) + { + throw new NotFoundException(); + } + return Response.ok(pkg).build(); + } + + @GET + @Path("searchFilename") + public Response getPackageByFileName(@QueryParam("file") String file) + { + TypedQuery fnquery = em.createQuery(fileNameQuery, PkgData.class); + fnquery.setParameter("fileName", file); + PkgData pkg = fnquery.getSingleResult(); + if (pkg == null) + { + throw new NotFoundException(); + } + return Response.ok(pkg).build(); + } + + @POST + @Path("/pkg/upload") + @Consumes(MediaType.APPLICATION_OCTET_STREAM) + public Response createPackage(byte[] input, @MatrixParam("filename") String filename) throws Exception + { + if (filename == null) + throw new BadRequestException(); + + String hash = Parser.computeMD5(new ByteArrayInputStream(input)); + TypedQuery hquery = em.createQuery(hashQuery, PkgData.class); + hquery.setParameter("md5sum", hash); + + List savedFiles = hquery.getResultList(); + + if (savedFiles.size() > 0) + { + return null; + } + else + { + File file = new File(prop.getProperty("RepoFolder"), filename); + FileOutputStream fos = new FileOutputStream(file); + fos.write(input); + fos.close(); + PkgData pkg = Parser.parseFile(file); + em.persist(pkg); + URI pkgUri = uriInfo.getAbsolutePathBuilder().path(pkg.filePath).build(); + return Response.created(pkgUri).build(); + } + } + +} + +// ====================================== +// = Public Methods = +// ====================================== + +/** + * curl -X POST --data-binary "Science fiction comedy bookfalse1-84023-742-235412.5The Hitchhiker's Guide to the Galaxy" -H "Content-Type: application/xml" http://localhost:8080/chapter15-service-1.0/rs/book -v + * curl -X POST --data-binary "{\"description\":\"Science fiction comedy book\",\"illustrations\":false,\"isbn\":\"1-84023-742-2\",\"nbOfPage\":354,\"price\":12.5,\"title\":\"The Hitchhiker's Guide to the Galaxy\"}" -H "Content-Type: application/json" http://localhost:8080/chapter15-service-1.0/rs/book -v + */ +// @POST +// public Response createBook(Book book) { +// if (book == null) +// throw new BadRequestException(); +// +// em.persist(book); +// URI bookUri = uriInfo.getAbsolutePathBuilder().path(book.getId()).build(); +// return Response.created(bookUri).build(); +// } + +// @PUT +// public Response updateBook(Book book) { +// if (book == null) +// throw new BadRequestException(); +// +// em.merge(book); +// return Response.ok().build(); +// } + +/** + * curl -X DELETE http://localhost:8080/chapter15-service-1.0/rs/book/1 -v + */ +// @DELETE +// @Path("{id}") +// public Response deleteBook(@PathParam("id") String id) { +// Book book = em.find(Book.class, id); +// +// if (book == null) +// throw new NotFoundException(); +// +// em.remove(book); +// +// return Response.noContent().build(); +// } + +/** + * JSON : curl -X GET -H "Accept: application/json" http://localhost:8080/chapter15-service-1.0/rs/book -v + * XML : curl -X GET -H "Accept: application/xml" http://localhost:8080/chapter15-service-1.0/rs/book -v + */ +// @GET +// public Response getAllBooks() { +// TypedQuery query = em.createNamedQuery(Book.FIND_ALL, Book.class); +// Books books = new Books(query.getResultList()); +// return Response.ok(books).build(); +// } \ No newline at end of file diff --git a/src/main/resources/META-INF/persistence.xml b/src/main/resources/META-INF/persistence.xml new file mode 100644 index 0000000..e76839c --- /dev/null +++ b/src/main/resources/META-INF/persistence.xml @@ -0,0 +1,17 @@ + + + + + java:/ejb/postgres/wildfly + + + + + + + + + diff --git a/src/test/java/ParseTest.java b/src/test/java/ParseTest.java new file mode 100644 index 0000000..30f1602 --- /dev/null +++ b/src/test/java/ParseTest.java @@ -0,0 +1,38 @@ +import com.thoughtworks.xstream.XStream; +import model.PkgData; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.RegexFileFilter; +import org.junit.Test; +import pacbase.Parser; + +import java.io.File; +import java.io.FileInputStream; +import java.nio.charset.Charset; +import java.util.*; + +/** + * Created by walter on 22/03/15. + */ +public class ParseTest +{ + @Test + public void test() throws Exception + { + + Collection ls = FileUtils.listFiles(new File("/var/cache/pacman/pkg"), new RegexFileFilter(".*\\.pkg\\.tar\\.xz"), DirectoryFileFilter.DIRECTORY); + int i=0; + List lista = new ArrayList<>(); + for(File file : ls) + { + PkgData data = Parser.parseFile(file); + lista.add(data); + //System.out.println(new XStream().toXML(data)); +// if(i++>10) break; + } + System.out.print(lista); + } +}