fixed bug with computing MD5 hash and multithreading

src/main/java/org/jpacrepo/pacbase/Hasher.java

@@ -1,9 +1,8 @@
 package org.jpacrepo.pacbase;
 
-import org.apache.commons.codec.binary.Hex;
-
 import java.io.IOException;
 import java.io.InputStream;
+import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
@@ -14,6 +13,8 @@ public class Hasher
 {
     static private final MessageDigest md5;
 
+    private MessageDigest md;
+
     static
     {
         try
@@ -25,20 +26,44 @@ public class Hasher
         }
     }
 
+    public Hasher(String algorithm)
+    {
+        try
+        {
+            md = MessageDigest.getInstance(algorithm);
+        }
+        catch (NoSuchAlgorithmException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
     public static String computeMD5(InputStream is) throws IOException
     {
-        md5.reset();
+        byte[] buffer = new byte[100000];
-        byte[] buffer2 = new byte[1000000];
         while (is.available() > 0)
         {
-            int read = is.read(buffer2, 0, 1000000);
+            int read = is.read(buffer, 0, 100000);
-            md5.update(buffer2,0,read);
+            md5.update(buffer, 0, read);
         }
         is.close();
         return bytesToHex(md5.digest());
     }
 
-    final protected static char[] hexArray = "0123456789ABCDEF".toCharArray();
+    public String getHashString(InputStream is) throws IOException
+    {
+        byte[] buffer = new byte[100000];
+        while (is.available() > 0)
+        {
+            int read = is.read(buffer, 0, 100000);
+            md.update(buffer, 0, read);
+        }
+        is.close();
+        return bytesToHex(md.digest());
+    }
+
+
+    final private static char[] hexArray = "0123456789ABCDEF".toCharArray();
 
     public static String bytesToHex(byte[] bytes)
     {
@@ -51,4 +76,21 @@ public class Hasher
         }
         return new String(hexChars);
     }
+
+    static MessageDigest getMd5()
+    {
+        try
+        {
+            return MessageDigest.getInstance("MD5");
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static MD5InputStream createMD5InputStream(InputStream is)
+    {
+        return new MD5InputStream(is);
+    }
 }

src/main/java/org/jpacrepo/pacbase/MD5InputStream.java

@@ -0,0 +1,21 @@
+package org.jpacrepo.pacbase;
+
+import java.io.InputStream;
+import java.security.DigestInputStream;
+
+/**
+ * Created by walter on 25/04/15.
+ */
+public class MD5InputStream extends DigestInputStream
+{
+    MD5InputStream(InputStream is)
+    {
+        super(is, Hasher.getMd5());
+    }
+
+    public String digest()
+    {
+        byte[] digest = getMessageDigest().digest();
+        return Hasher.bytesToHex(digest);
+    }
+}

src/main/java/org/jpacrepo/pacbase/Parser.java

@@ -17,12 +17,20 @@ import java.util.*;
 public class Parser
 {
 
-    public static PkgData parseFile(File file) throws Exception
+    private XZCompressorInputStream xzcis;
+    private TarArchiveInputStream tais;
+    private ArchiveEntry ae;
+    private Hasher hasher;
+
+    public Parser()
     {
-        FileInputStream fis = new FileInputStream(file);
+        hasher = new Hasher("MD5");
-        XZCompressorInputStream xzcis = new XZCompressorInputStream(fis);
+    }
-        TarArchiveInputStream tais = new TarArchiveInputStream(xzcis);
+
-        ArchiveEntry ae;
+    public PkgData parseFile(File file) throws Exception
+    {
+        xzcis = new XZCompressorInputStream(new FileInputStream(file));
+        tais = new TarArchiveInputStream(xzcis);
         while((ae = tais.getNextEntry()) != null)
         {
             if(ae.getName().equals(".PKGINFO"))
@@ -57,7 +65,6 @@ public class Parser
                 }
                 tais.close();
                 xzcis.close();
-                fis.close();
                 PkgData data = new PkgData();
 
                 for(String key : propMap.keySet())
@@ -118,17 +125,15 @@ public class Parser
                         case "pkgbase":
                             data.base = propMap.get(key).get(0);
                             break;
-
                     }
                 }
-                data.md5sum = Hasher.computeMD5(new FileInputStream(file));
+                data.md5sum = hasher.getHashString(new FileInputStream(file));
                 data.fileName = file.getName();
                 return data;
             }
         }
         tais.close();
         xzcis.close();
-        fis.close();
         return null;
     }
 

src/main/java/org/jpacrepo/service/PacmanServiceEJB.java

@@ -108,12 +108,17 @@ public class PacmanServiceEJB implements PacmanService
         }
     }
 
-    private void parseFile(File file) throws Exception
+    private void parseFile2(Parser p, File file) throws Exception
+    {
+
+    }
+
+    private void parseFile(Parser p, File file) throws Exception
     {
         TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
         TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
 
-        PkgData data = Parser.parseFile(file);
+        PkgData data = p.parseFile(file);
         hquery.setParameter("md5sum", data.md5sum);
         List<PkgData>  savedFiles = hquery.getResultList();
         if (savedFiles.size() > 0)
@@ -172,6 +177,8 @@ public class PacmanServiceEJB implements PacmanService
     {
         Deque<File> ls;
 
+        Parser parser = new Parser();
+
         public SyncWorker(Deque<File> ls)
         {
             this.ls = ls;
@@ -193,7 +200,7 @@ public class PacmanServiceEJB implements PacmanService
                     if (p == null || file.lastModified() > p.updTimestamp.getTime())
                     {
                         ut.begin();
-                        parseFile(file);
+                        parseFile(parser, file);
                         ut.commit();
                     }
                 }

src/main/java/org/jpacrepo/service/PacmanWebService.java

@@ -7,7 +7,6 @@ import org.jpacrepo.model.PkgData;
 import org.jpacrepo.model.PkgList;
 import org.jpacrepo.model.PkgName;
 import org.apache.commons.io.IOUtils;
-import org.jpacrepo.pacbase.Hasher;
 import org.jpacrepo.pacbase.Parser;
 import org.jpacrepo.persistence.QueryEngine;
 
@@ -20,7 +19,6 @@ import javax.ws.rs.core.Context;
 import java.io.*;
 import java.net.URI;
 import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -49,6 +47,8 @@ public class PacmanWebService
     @DefaultConfiguration
     private ApplicationContext ctx;
 
+    private Parser serviceParser = new Parser();
+
     @GET
     @Path("search")
     public Response getPackage(@QueryParam("name") String name,
@@ -184,7 +184,7 @@ public class PacmanWebService
         }
         else
         {
-            PkgData pkg = Parser.parseFile(file);
+            PkgData pkg = serviceParser.parseFile(file);
 
             TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
             hquery.setParameter("md5sum", pkg.md5sum);

src/main/resources/META-INF/persistence.xml

@@ -7,8 +7,8 @@
   <persistence-unit name="jpacrepo_pu" transaction-type="JTA">
     <jta-data-source>java:/ejb/postgres/wildfly</jta-data-source>
     <properties>
-      <property name="javax.persistence.schema-generation.database.action" value="drop-and-create"/>
+      <!--<property name="javax.persistence.schema-generation.database.action" value="drop-and-create"/>-->
-        <!--<property name="javax.persistence.schema-generation.database.action" value="none"/>-->
+        <property name="javax.persistence.schema-generation.database.action" value="none"/>
         <!--<property name="javax.persistence.schema-generation.database.action" value="create"/>-->
       <property name="eclipselink.logging.level" value="INFO"/>
       <property name="hibernate.default_schema" value="jpacrepo"/>

src/test/java/ClientTest.java

@@ -1,17 +1,24 @@
 import com.thoughtworks.xstream.XStream;
-import org.jpacrepo.model.PkgData;
 import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
 import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
 import org.jboss.resteasy.spi.ResteasyProviderFactory;
+import org.jpacrepo.model.PkgData;
+import org.jpacrepo.pacbase.Hasher;
+import org.jpacrepo.pacbase.MD5InputStream;
+import org.jpacrepo.pacbase.Parser;
 import org.junit.Test;
 
 import javax.ws.rs.client.*;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
 
 /**
  * Created by walter on 29/03/15.
@@ -57,4 +64,39 @@ public class ClientTest
         System.out.println(response.getStatusInfo());
         assert Response.Status.CREATED.getStatusCode() == response.getStatus();
     }
+
+    @Test
+    public void hashTest() throws Exception
+    {
+        String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
+
+        for (String file : files)
+        {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            try (InputStream is = Files.newInputStream(Paths.get(file)))
+            {
+                DigestInputStream dis = new DigestInputStream(is, md);
+                dis.on(true);
+//                is.read();
+                long a = new File(file).length();
+                byte[] out = new byte[(int)a];
+                dis.read(out, 0, (int) a);
+            }
+            System.out.println(Hasher.bytesToHex(md.digest()));
+
+            System.out.println(Hasher.computeMD5(new FileInputStream(file)));
+
+            InputStream fis = new FileInputStream(file);
+            MD5InputStream h = Hasher.createMD5InputStream(fis);
+            long a = new File(file).length();
+            byte[] out = new byte[(int)a];
+            h.read(out, 0, (int) a);
+            System.out.println(h.digest());
+
+            PkgData p = new Parser().parseFile(new File(file));
+            System.out.println(p.md5sum);
+
+        }
+
+    }
 }

src/test/java/ParseTest.java

@@ -1,23 +1,26 @@
-import org.jpacrepo.model.PkgData;
+import com.thoughtworks.xstream.XStream;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.DirectoryFileFilter;
 import org.apache.commons.io.filefilter.RegexFileFilter;
-import org.junit.Test;
+import org.jpacrepo.model.PkgData;
 import org.jpacrepo.pacbase.Parser;
 import org.jpacrepo.service.PacmanService;
+import org.junit.Test;
 
 import javax.naming.*;
 import java.io.File;
-import java.io.IOException;
 import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
 
 /**
  * Created by walter on 22/03/15.
  */
 public class ParseTest
 {
-//    @Test
+    //    @Test
     public void test() throws Exception
     {
 
@@ -26,7 +29,7 @@ public class ParseTest
         List<PkgData> lista = new ArrayList<>();
         for (File file : ls)
         {
-            PkgData data = Parser.parseFile(file);
+            PkgData data = new Parser().parseFile(file);
             lista.add(data);
             //System.out.println(new XStream().toXML(data));
 //            if(i++>10) break;
@@ -59,16 +62,32 @@ public class ParseTest
         stateService.deletePackage("linux-3.19.3-3-x86_64.pkg.tar.xz");
     }
 
-    private static void traverseJndiNode(String nodeName, Context context) {
+    private static void traverseJndiNode(String nodeName, Context context)
-        try {
+    {
+        try
+        {
             NamingEnumeration<NameClassPair> list = context.list(nodeName);
-            while (list.hasMore()){
+            while (list.hasMore())
+            {
                 String childName = nodeName + "" + list.next().getName();
                 System.out.println(childName);
                 traverseJndiNode(childName, context);
             }
-        } catch (NamingException ex) {
+        } catch (NamingException ex)
+        {
 // We reached a leaf
         }
     }
+
+    @Test
+    public void parseTest() throws Exception
+    {
+        String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
+
+        for (String file : files)
+        {
+            PkgData data = new Parser().parseFile(new File(file));
+            System.out.println(new XStream().toXML(data));
+        }
+    }
 }