woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

fixed bug with computing MD5 hash and multithreading

Browse Source
This commit is contained in:
Walter Oggioni
2015-04-25 17:50:26 +02:00
parent db786b73c9
commit ee86dfe895
8 changed files with 172 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/org/jpacrepo/pacbase/Hasher.java
View File

@@ -1,9 +1,8 @@
package org.jpacrepo.pacbase;
import org.apache.commons.codec.binary.Hex;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -14,6 +13,8 @@ public class Hasher
{
static private final MessageDigest md5;
private MessageDigest md;
static
{
try
@@ -25,20 +26,44 @@ public class Hasher
}
}
public Hasher(String algorithm)
{
try
{
md = MessageDigest.getInstance(algorithm);
}
catch (NoSuchAlgorithmException e)
{
throw new RuntimeException(e);
}
}
public static String computeMD5(InputStream is) throws IOException
{
md5.reset();
byte[] buffer2 = new byte[1000000];
byte[] buffer = new byte[100000];
while (is.available() > 0)
{
int read = is.read(buffer2, 0, 1000000);
md5.update(buffer2,0,read);
int read = is.read(buffer, 0, 100000);
md5.update(buffer, 0, read);
}
is.close();
return bytesToHex(md5.digest());
}
final protected static char[] hexArray = "0123456789ABCDEF".toCharArray();
public String getHashString(InputStream is) throws IOException
{
byte[] buffer = new byte[100000];
while (is.available() > 0)
{
int read = is.read(buffer, 0, 100000);
md.update(buffer, 0, read);
}
is.close();
return bytesToHex(md.digest());
}
final private static char[] hexArray = "0123456789ABCDEF".toCharArray();
public static String bytesToHex(byte[] bytes)
{
@@ -51,4 +76,21 @@ public class Hasher
}
return new String(hexChars);
}
static MessageDigest getMd5()
{
try
{
return MessageDigest.getInstance("MD5");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public static MD5InputStream createMD5InputStream(InputStream is)
{
return new MD5InputStream(is);
}
}

21
src/main/java/org/jpacrepo/pacbase/MD5InputStream.java Normal file
View File

@@ -0,0 +1,21 @@
package org.jpacrepo.pacbase;
import java.io.InputStream;
import java.security.DigestInputStream;
/**
* Created by walter on 25/04/15.
*/
public class MD5InputStream extends DigestInputStream
{
MD5InputStream(InputStream is)
{
super(is, Hasher.getMd5());
}
public String digest()
{
byte[] digest = getMessageDigest().digest();
return Hasher.bytesToHex(digest);
}
}

23
src/main/java/org/jpacrepo/pacbase/Parser.java
View File

@@ -17,12 +17,20 @@ import java.util.*;
public class Parser
{
public static PkgData parseFile(File file) throws Exception
private XZCompressorInputStream xzcis;
private TarArchiveInputStream tais;
private ArchiveEntry ae;
private Hasher hasher;
public Parser()
{
FileInputStream fis = new FileInputStream(file);
XZCompressorInputStream xzcis = new XZCompressorInputStream(fis);
TarArchiveInputStream tais = new TarArchiveInputStream(xzcis);
ArchiveEntry ae;
hasher = new Hasher("MD5");
}
public PkgData parseFile(File file) throws Exception
{
xzcis = new XZCompressorInputStream(new FileInputStream(file));
tais = new TarArchiveInputStream(xzcis);
while((ae = tais.getNextEntry()) != null)
{
if(ae.getName().equals(".PKGINFO"))
@@ -57,7 +65,6 @@ public class Parser
}
tais.close();
xzcis.close();
fis.close();
PkgData data = new PkgData();
for(String key : propMap.keySet())
@@ -118,17 +125,15 @@ public class Parser
case "pkgbase":
data.base = propMap.get(key).get(0);
break;
}
}
data.md5sum = Hasher.computeMD5(new FileInputStream(file));
data.md5sum = hasher.getHashString(new FileInputStream(file));
data.fileName = file.getName();
return data;
}
}
tais.close();
xzcis.close();
fis.close();
return null;
}

13
src/main/java/org/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -108,12 +108,17 @@ public class PacmanServiceEJB implements PacmanService
}
}
private void parseFile(File file) throws Exception
private void parseFile2(Parser p, File file) throws Exception
{
}
private void parseFile(Parser p, File file) throws Exception
{
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
PkgData data = Parser.parseFile(file);
PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum);
List<PkgData> savedFiles = hquery.getResultList();
if (savedFiles.size() > 0)
@@ -172,6 +177,8 @@ public class PacmanServiceEJB implements PacmanService
{
Deque<File> ls;
Parser parser = new Parser();
public SyncWorker(Deque<File> ls)
{
this.ls = ls;
@@ -193,7 +200,7 @@ public class PacmanServiceEJB implements PacmanService
if (p == null || file.lastModified() > p.updTimestamp.getTime())
{
ut.begin();
parseFile(file);
parseFile(parser, file);
ut.commit();
}
}

6
src/main/java/org/jpacrepo/service/PacmanWebService.java
View File

@@ -7,7 +7,6 @@ import org.jpacrepo.model.PkgData;
import org.jpacrepo.model.PkgList;
import org.jpacrepo.model.PkgName;
import org.apache.commons.io.IOUtils;
import org.jpacrepo.pacbase.Hasher;
import org.jpacrepo.pacbase.Parser;
import org.jpacrepo.persistence.QueryEngine;
@@ -20,7 +19,6 @@ import javax.ws.rs.core.Context;
import java.io.*;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -49,6 +47,8 @@ public class PacmanWebService
@DefaultConfiguration
private ApplicationContext ctx;
private Parser serviceParser = new Parser();
@GET
@Path("search")
public Response getPackage(@QueryParam("name") String name,
@@ -184,7 +184,7 @@ public class PacmanWebService
}
else
{
PkgData pkg = Parser.parseFile(file);
PkgData pkg = serviceParser.parseFile(file);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
hquery.setParameter("md5sum", pkg.md5sum);

4
src/main/resources/META-INF/persistence.xml
View File

@@ -7,8 +7,8 @@
<persistence-unit name="jpacrepo_pu" transaction-type="JTA">
<jta-data-source>java:/ejb/postgres/wildfly</jta-data-source>
<properties>
<property name="javax.persistence.schema-generation.database.action" value="drop-and-create"/>
<!--<property name="javax.persistence.schema-generation.database.action" value="none"/>-->
<!--<property name="javax.persistence.schema-generation.database.action" value="drop-and-create"/>-->
<property name="javax.persistence.schema-generation.database.action" value="none"/>
<!--<property name="javax.persistence.schema-generation.database.action" value="create"/>-->
<property name="eclipselink.logging.level" value="INFO"/>
<property name="hibernate.default_schema" value="jpacrepo"/>

46
src/test/java/ClientTest.java
View File

@@ -1,17 +1,24 @@
import com.thoughtworks.xstream.XStream;
import org.jpacrepo.model.PkgData;
import org.jboss.resteasy.plugins.providers.RegisterBuiltin;
import org.jboss.resteasy.plugins.providers.jackson.ResteasyJacksonProvider;
import org.jboss.resteasy.spi.ResteasyProviderFactory;
import org.jpacrepo.model.PkgData;
import org.jpacrepo.pacbase.Hasher;
import org.jpacrepo.pacbase.MD5InputStream;
import org.jpacrepo.pacbase.Parser;
import org.junit.Test;
import javax.ws.rs.client.*;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.MessageDigest;
/**
* Created by walter on 29/03/15.
@@ -57,4 +64,39 @@ public class ClientTest
System.out.println(response.getStatusInfo());
assert Response.Status.CREATED.getStatusCode() == response.getStatus();
}
@Test
public void hashTest() throws Exception
{
String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
for (String file : files)
{
MessageDigest md = MessageDigest.getInstance("MD5");
try (InputStream is = Files.newInputStream(Paths.get(file)))
{
DigestInputStream dis = new DigestInputStream(is, md);
dis.on(true);
// is.read();
long a = new File(file).length();
byte[] out = new byte[(int)a];
dis.read(out, 0, (int) a);
}
System.out.println(Hasher.bytesToHex(md.digest()));
System.out.println(Hasher.computeMD5(new FileInputStream(file)));
InputStream fis = new FileInputStream(file);
MD5InputStream h = Hasher.createMD5InputStream(fis);
long a = new File(file).length();
byte[] out = new byte[(int)a];
h.read(out, 0, (int) a);
System.out.println(h.digest());
PkgData p = new Parser().parseFile(new File(file));
System.out.println(p.md5sum);
}
}
}

39
src/test/java/ParseTest.java
View File

@@ -1,23 +1,26 @@
import org.jpacrepo.model.PkgData;
import com.thoughtworks.xstream.XStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.junit.Test;
import org.jpacrepo.model.PkgData;
import org.jpacrepo.pacbase.Parser;
import org.jpacrepo.service.PacmanService;
import org.junit.Test;
import javax.naming.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
/**
* Created by walter on 22/03/15.
*/
public class ParseTest
{
// @Test
// @Test
public void test() throws Exception
{
@@ -26,7 +29,7 @@ public class ParseTest
List<PkgData> lista = new ArrayList<>();
for (File file : ls)
{
PkgData data = Parser.parseFile(file);
PkgData data = new Parser().parseFile(file);
lista.add(data);
//System.out.println(new XStream().toXML(data));
// if(i++>10) break;
@@ -59,16 +62,32 @@ public class ParseTest
stateService.deletePackage("linux-3.19.3-3-x86_64.pkg.tar.xz");
}
private static void traverseJndiNode(String nodeName, Context context) {
try {
private static void traverseJndiNode(String nodeName, Context context)
{
try
{
NamingEnumeration<NameClassPair> list = context.list(nodeName);
while (list.hasMore()){
while (list.hasMore())
{
String childName = nodeName + "" + list.next().getName();
System.out.println(childName);
traverseJndiNode(childName, context);
}
} catch (NamingException ex) {
} catch (NamingException ex)
{
// We reached a leaf
}
}
@Test
public void parseTest() throws Exception
{
String[] files = new String[]{"/var/cache/pacman/pkg/mesa-10.4.5-1-x86_64.pkg.tar.xz", "/var/cache/pacman/pkg/mesa-10.5.3-1-x86_64.pkg.tar.xz"};
for (String file : files)
{
PkgData data = new Parser().parseFile(new File(file));
System.out.println(new XStream().toXML(data));
}
}
}