diff --git a/src/main/java/org/jpacrepo/pacbase/Hasher.java b/src/main/java/org/jpacrepo/pacbase/Hasher.java index ce9c800..74f6c5b 100644 --- a/src/main/java/org/jpacrepo/pacbase/Hasher.java +++ b/src/main/java/org/jpacrepo/pacbase/Hasher.java @@ -1,9 +1,8 @@ package org.jpacrepo.pacbase; -import org.apache.commons.codec.binary.Hex; - import java.io.IOException; import java.io.InputStream; +import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -14,6 +13,8 @@ public class Hasher { static private final MessageDigest md5; + private MessageDigest md; + static { try @@ -25,20 +26,44 @@ public class Hasher } } + public Hasher(String algorithm) + { + try + { + md = MessageDigest.getInstance(algorithm); + } + catch (NoSuchAlgorithmException e) + { + throw new RuntimeException(e); + } + } + public static String computeMD5(InputStream is) throws IOException { - md5.reset(); - byte[] buffer2 = new byte[1000000]; + byte[] buffer = new byte[100000]; while (is.available() > 0) { - int read = is.read(buffer2, 0, 1000000); - md5.update(buffer2,0,read); + int read = is.read(buffer, 0, 100000); + md5.update(buffer, 0, read); } is.close(); return bytesToHex(md5.digest()); } - final protected static char[] hexArray = "0123456789ABCDEF".toCharArray(); + public String getHashString(InputStream is) throws IOException + { + byte[] buffer = new byte[100000]; + while (is.available() > 0) + { + int read = is.read(buffer, 0, 100000); + md.update(buffer, 0, read); + } + is.close(); + return bytesToHex(md.digest()); + } + + + final private static char[] hexArray = "0123456789ABCDEF".toCharArray(); public static String bytesToHex(byte[] bytes) { @@ -51,4 +76,21 @@ public class Hasher } return new String(hexChars); } + + static MessageDigest getMd5() + { + try + { + return MessageDigest.getInstance("MD5"); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + public static MD5InputStream createMD5InputStream(InputStream is) + { + return new MD5InputStream(is); + } } diff --git a/src/main/java/org/jpacrepo/pacbase/MD5InputStream.java b/src/main/java/org/jpacrepo/pacbase/MD5InputStream.java new file mode 100644 index 0000000..4912a65 --- /dev/null +++ b/src/main/java/org/jpacrepo/pacbase/MD5InputStream.java @@ -0,0 +1,21 @@ +package org.jpacrepo.pacbase; + +import java.io.InputStream; +import java.security.DigestInputStream; + +/** + * Created by walter on 25/04/15. + */ +public class MD5InputStream extends DigestInputStream +{ + MD5InputStream(InputStream is) + { + super(is, Hasher.getMd5()); + } + + public String digest() + { + byte[] digest = getMessageDigest().digest(); + return Hasher.bytesToHex(digest); + } +} diff --git a/src/main/java/org/jpacrepo/pacbase/Parser.java b/src/main/java/org/jpacrepo/pacbase/Parser.java index d084682..2a43782 100644 --- a/src/main/java/org/jpacrepo/pacbase/Parser.java +++ b/src/main/java/org/jpacrepo/pacbase/Parser.java @@ -17,12 +17,20 @@ import java.util.*; public class Parser { - public static PkgData parseFile(File file) throws Exception + private XZCompressorInputStream xzcis; + private TarArchiveInputStream tais; + private ArchiveEntry ae; + private Hasher hasher; + + public Parser() { - FileInputStream fis = new FileInputStream(file); - XZCompressorInputStream xzcis = new XZCompressorInputStream(fis); - TarArchiveInputStream tais = new TarArchiveInputStream(xzcis); - ArchiveEntry ae; + hasher = new Hasher("MD5"); + } + + public PkgData parseFile(File file) throws Exception + { + xzcis = new XZCompressorInputStream(new FileInputStream(file)); + tais = new TarArchiveInputStream(xzcis); while((ae = tais.getNextEntry()) != null) { if(ae.getName().equals(".PKGINFO")) @@ -57,7 +65,6 @@ public class Parser } tais.close(); xzcis.close(); - fis.close(); PkgData data = new PkgData(); for(String key : propMap.keySet()) @@ -118,17 +125,15 @@ public class Parser case "pkgbase": data.base = propMap.get(key).get(0); break; - } } - data.md5sum = Hasher.computeMD5(new FileInputStream(file)); + data.md5sum = hasher.getHashString(new FileInputStream(file)); data.fileName = file.getName(); return data; } } tais.close(); xzcis.close(); - fis.close(); return null; } diff --git a/src/main/java/org/jpacrepo/service/PacmanServiceEJB.java b/src/main/java/org/jpacrepo/service/PacmanServiceEJB.java index 037ff56..bfbe2c7 100644 --- a/src/main/java/org/jpacrepo/service/PacmanServiceEJB.java +++ b/src/main/java/org/jpacrepo/service/PacmanServiceEJB.java @@ -108,12 +108,17 @@ public class PacmanServiceEJB implements PacmanService } } - private void parseFile(File file) throws Exception + private void parseFile2(Parser p, File file) throws Exception + { + + } + + private void parseFile(Parser p, File file) throws Exception { TypedQuery nquery = em.createQuery(nameQuery, PkgName.class); TypedQuery hquery = em.createQuery(hashQuery, PkgData.class); - PkgData data = Parser.parseFile(file); + PkgData data = p.parseFile(file); hquery.setParameter("md5sum", data.md5sum); List savedFiles = hquery.getResultList(); if (savedFiles.size() > 0) @@ -172,6 +177,8 @@ public class PacmanServiceEJB implements PacmanService { Deque ls; + Parser parser = new Parser(); + public SyncWorker(Deque ls) { this.ls = ls; @@ -193,7 +200,7 @@ public class PacmanServiceEJB implements PacmanService if (p == null || file.lastModified() > p.updTimestamp.getTime()) { ut.begin(); - parseFile(file); + parseFile(parser, file); ut.commit(); } } diff --git a/src/main/java/org/jpacrepo/service/PacmanWebService.java b/src/main/java/org/jpacrepo/service/PacmanWebService.java index 67be12e..a6eeb0c 100644 --- a/src/main/java/org/jpacrepo/service/PacmanWebService.java +++ b/src/main/java/org/jpacrepo/service/PacmanWebService.java @@ -7,7 +7,6 @@ import org.jpacrepo.model.PkgData; import org.jpacrepo.model.PkgList; import org.jpacrepo.model.PkgName; import org.apache.commons.io.IOUtils; -import org.jpacrepo.pacbase.Hasher; import org.jpacrepo.pacbase.Parser; import org.jpacrepo.persistence.QueryEngine; @@ -20,7 +19,6 @@ import javax.ws.rs.core.Context; import java.io.*; import java.net.URI; import java.nio.file.Files; -import java.nio.file.StandardCopyOption; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -49,6 +47,8 @@ public class PacmanWebService @DefaultConfiguration private ApplicationContext ctx; + private Parser serviceParser = new Parser(); + @GET @Path("search") public Response getPackage(@QueryParam("name") String name, @@ -184,7 +184,7 @@ public class PacmanWebService } else { - PkgData pkg = Parser.parseFile(file); + PkgData pkg = serviceParser.parseFile(file); TypedQuery hquery = em.createQuery(hashQuery, PkgData.class); hquery.setParameter("md5sum", pkg.md5sum); diff --git a/src/main/resources/META-INF/persistence.xml b/src/main/resources/META-INF/persistence.xml index 33c0a11..f5aae31 100644 --- a/src/main/resources/META-INF/persistence.xml +++ b/src/main/resources/META-INF/persistence.xml @@ -7,8 +7,8 @@ java:/ejb/postgres/wildfly - - + + diff --git a/src/test/java/ClientTest.java b/src/test/java/ClientTest.java index 4b4f6d3..4f5f6d6 100644 --- a/src/test/java/ClientTest.java +++ b/src/test/java/ClientTest.java @@ -1,17 +1,24 @@ import com.thoughtworks.xstream.XStream; -import org.jpacrepo.model.PkgData; import org.jboss.resteasy.plugins.providers.RegisterBuiltin; import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider; import org.jboss.resteasy.spi.ResteasyProviderFactory; +import org.jpacrepo.model.PkgData; +import org.jpacrepo.pacbase.Hasher; +import org.jpacrepo.pacbase.MD5InputStream; +import org.jpacrepo.pacbase.Parser; import org.junit.Test; import javax.ws.rs.client.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.DigestInputStream; +import java.security.MessageDigest; /** * Created by walter on 29/03/15. @@ -57,4 +64,39 @@ public class ClientTest System.out.println(response.getStatusInfo()); assert Response.Status.CREATED.getStatusCode() == response.getStatus(); } + + @Test + public void hashTest() throws Exception + { + String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"}; + + for (String file : files) + { + MessageDigest md = MessageDigest.getInstance("MD5"); + try (InputStream is = Files.newInputStream(Paths.get(file))) + { + DigestInputStream dis = new DigestInputStream(is, md); + dis.on(true); +// is.read(); + long a = new File(file).length(); + byte[] out = new byte[(int)a]; + dis.read(out, 0, (int) a); + } + System.out.println(Hasher.bytesToHex(md.digest())); + + System.out.println(Hasher.computeMD5(new FileInputStream(file))); + + InputStream fis = new FileInputStream(file); + MD5InputStream h = Hasher.createMD5InputStream(fis); + long a = new File(file).length(); + byte[] out = new byte[(int)a]; + h.read(out, 0, (int) a); + System.out.println(h.digest()); + + PkgData p = new Parser().parseFile(new File(file)); + System.out.println(p.md5sum); + + } + + } } diff --git a/src/test/java/ParseTest.java b/src/test/java/ParseTest.java index 84efa3a..d1543c1 100644 --- a/src/test/java/ParseTest.java +++ b/src/test/java/ParseTest.java @@ -1,23 +1,26 @@ -import org.jpacrepo.model.PkgData; +import com.thoughtworks.xstream.XStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.io.filefilter.RegexFileFilter; -import org.junit.Test; +import org.jpacrepo.model.PkgData; import org.jpacrepo.pacbase.Parser; import org.jpacrepo.service.PacmanService; +import org.junit.Test; import javax.naming.*; import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; /** * Created by walter on 22/03/15. */ public class ParseTest { -// @Test + // @Test public void test() throws Exception { @@ -26,7 +29,7 @@ public class ParseTest List lista = new ArrayList<>(); for (File file : ls) { - PkgData data = Parser.parseFile(file); + PkgData data = new Parser().parseFile(file); lista.add(data); //System.out.println(new XStream().toXML(data)); // if(i++>10) break; @@ -59,16 +62,32 @@ public class ParseTest stateService.deletePackage("linux-3.19.3-3-x86_64.pkg.tar.xz"); } - private static void traverseJndiNode(String nodeName, Context context) { - try { + private static void traverseJndiNode(String nodeName, Context context) + { + try + { NamingEnumeration list = context.list(nodeName); - while (list.hasMore()){ + while (list.hasMore()) + { String childName = nodeName + "" + list.next().getName(); System.out.println(childName); traverseJndiNode(childName, context); } - } catch (NamingException ex) { + } catch (NamingException ex) + { // We reached a leaf } } + + @Test + public void parseTest() throws Exception + { + String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"}; + + for (String file : files) + { + PkgData data = new Parser().parseFile(new File(file)); + System.out.println(new XStream().toXML(data)); + } + } }