improved performances of initial sync

jpacrepo-impl/src/main/java/net/woggioni/jpacrepo/impl/model/PkgDataParser.java

@@ -32,6 +32,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.text.ParseException;
 import java.time.Instant;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -45,6 +46,9 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 
+import static net.woggioni.jwo.JWO.iterable2Stream;
+import static net.woggioni.jwo.JWO.not;
+
 
 @Slf4j
 @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
@@ -78,10 +82,22 @@ class NamedEntityFinder<T extends NamedEntity> {
         var query = em.createQuery(tq);
         query.setMaxResults(1);
         return Optional.of(query.getResultList())
-            .filter(JWO.not(List::isEmpty))
+            .filter(not(List::isEmpty))
             .map(it -> it.get(0));
     }
 
+    public Set<T> queryNamedEntities(Collection<String> names) {
+        if(log.isDebugEnabled()) {
+            log.debug("Querying named entities '{}' with names {}", cls.getName(), String.join(", ", names));
+        }
+        var cb = em.getCriteriaBuilder();
+        var tq = cb.createQuery(cls);
+        var root = tq.from(cls);
+        tq.select(root).where(root.get(NamedEntity_.NAME).in(names));
+        var query = em.createQuery(tq);
+        return query.getResultStream().collect(CollectionUtils.toUnmodifiableSet());
+    }
+
     public T getByName(String name) {
         return
             cache.computeIfAbsent(
@@ -94,6 +110,13 @@ class NamedEntityFinder<T extends NamedEntity> {
                     })
             );
     }
+
+    public void persistNewEntities(Iterable<T> entities) {
+        for(T entity : entities) {
+            em.persist(entity);
+            cache.put(entity.getName(), entity);
+        }
+    }
 }
 
 public class PkgDataParser {
@@ -127,6 +150,94 @@ public class PkgDataParser {
             .collect(Collectors.toUnmodifiableSet());
     }
 
+    public List<PkgBase> getNewPkgBases(Iterable<PkgData> pkgDatas) {
+        Set<PkgBase> allPackagers = iterable2Stream(pkgDatas)
+            .map(PkgData::getBase).collect(Collectors.toUnmodifiableSet());
+
+        Set<PkgBase> existingPackagers = pkgBaseFinder.queryNamedEntities(
+            allPackagers.stream()
+                .map(PkgBase::getName)
+                .collect(CollectionUtils.toUnmodifiableList())
+        );
+
+        return allPackagers
+            .stream()
+            .filter(not(existingPackagers::contains))
+            .toList();
+    }
+
+    public List<License> getNewLicenses(Iterable<PkgData> pkgDatas) {
+        Set<License> allLicenses = iterable2Stream(pkgDatas)
+            .map(PkgData::getLicense)
+            .flatMap(Set::stream)
+            .collect(Collectors.toUnmodifiableSet());
+
+        Set<License> existingLicenses = licenseFinder.queryNamedEntities(
+            allLicenses.stream()
+                .map(License::getName)
+                .collect(CollectionUtils.toUnmodifiableList())
+        );
+
+        return allLicenses
+            .stream()
+            .filter(not(existingLicenses::contains))
+            .toList();
+    }
+    public List<Packager> getNewPackagers(Iterable<PkgData> pkgDatas) {
+        Set<Packager> allPackagers = iterable2Stream(pkgDatas)
+            .map(PkgData::getPackager).collect(Collectors.toUnmodifiableSet());
+
+        Set<Packager> existingPackagers = packagerFinder.queryNamedEntities(
+            allPackagers.stream()
+                .map(Packager::getName)
+                .collect(CollectionUtils.toUnmodifiableList())
+        );
+
+        return allPackagers
+            .stream()
+            .filter(not(existingPackagers::contains))
+            .toList();
+    }
+
+    public List<Dependency> getNewDependencies(Iterable<PkgData> pkgDatas) {
+        Set<Dependency> allDependencies = iterable2Stream(pkgDatas)
+            .flatMap(pkgData -> JWO.streamCat(
+                    pkgData.getDepend().stream(),
+                    pkgData.getMakedepend().stream(),
+                    pkgData.getMakepkgopt().stream(),
+                    pkgData.getOptdepend().stream(),
+                    pkgData.getBackup().stream(),
+                    pkgData.getConflict().stream(),
+                    pkgData.getReplaces().stream(),
+                    pkgData.getProvides().stream()
+                )
+            ).collect(Collectors.toUnmodifiableSet());
+
+        Set<Dependency> existingDeps = dependencyFinder.queryNamedEntities(
+            allDependencies.stream()
+                .map(Dependency::getName)
+                .collect(CollectionUtils.toUnmodifiableList())
+        );
+
+        return allDependencies
+            .stream()
+            .filter(not(existingDeps::contains))
+            .toList();
+    }
+
+    public void addNewDependencies(Iterable<Dependency> dependencies) {
+        dependencyFinder.persistNewEntities(dependencies);
+    }
+    public void addNewPackagers(Iterable<Packager> packagers) {
+        packagerFinder.persistNewEntities(packagers);
+    }
+    public void addNewPkgBases(Iterable<PkgBase> pkgBases) {
+        pkgBaseFinder.persistNewEntities(pkgBases);
+    }
+    public void addNewLicenses(Iterable<License> licenses) {
+        licenseFinder.persistNewEntities(licenses);
+    }
+
     public PkgData hydrateJPA(PkgData pkgData) {
         pkgData.setBase(hydrate(pkgBaseFinder, pkgData.getBase()));
         pkgData.setDepend(hydrate(dependencyFinder, pkgData.getDepend()));