Sort only if we count, and sort by highest frequency first

See https://gitlab.com/oersi/oersi-etl/-/issues/238
metafacture · May 5, 2023 · 17b836d · 17b836d
1 parent f906faf
commit 17b836d
Show file tree

Hide file tree

Showing 2 changed files with 78 additions and 27 deletions.
diff --git a/metafix/src/main/java/org/metafacture/metafix/MetafixListPaths.java b/metafix/src/main/java/org/metafacture/metafix/MetafixListPaths.java
@@ -18,12 +18,15 @@
 
 import org.metafacture.formatting.ObjectTemplate;
 import org.metafacture.framework.ObjectReceiver;
+import org.metafacture.framework.Sender;
 import org.metafacture.framework.helpers.DefaultStreamPipe;
-import org.metafacture.framework.objects.Triple;
+import org.metafacture.mangling.DuplicateObjectFilter;
 import org.metafacture.mangling.StreamFlattener;
 import org.metafacture.triples.AbstractTripleSort.Compare;
+import org.metafacture.triples.AbstractTripleSort.Order;
 import org.metafacture.triples.StreamToTriples;
 import org.metafacture.triples.TripleCount;
+import org.metafacture.triples.TripleSort;
 
 import java.io.FileNotFoundException;
 
@@ -50,17 +53,39 @@ public MetafixListPaths() {
 
     @Override
     protected void onSetReceiver() {
+        fix.setEntityMemberName(index ? "%d" : "*");
+        final StreamToTriples triples = fix
+                .setReceiver(new StreamFlattener())
+                .setReceiver(new StreamToTriples());
+        (count ? counted(triples) : unique(triples))
+                .setReceiver(getReceiver());
+    }
+
+    private Sender<ObjectReceiver<String>> counted(final StreamToTriples triples) {
+        return triples
+                .setReceiver(tripleCount())
+                .setReceiver(tripleSort())
+                .setReceiver(new ObjectTemplate<>("${s}\t ${o}"));
+    }
+
+    private Sender<ObjectReceiver<String>> unique(final StreamToTriples triples) {
+        return triples
+                .setReceiver(new ObjectTemplate<>("${p}"))
+                .setReceiver(new DuplicateObjectFilter<>());
+    }
+
+    private TripleCount tripleCount() {
         final TripleCount tripleCount = new TripleCount();
         tripleCount.setCountBy(Compare.PREDICATE);
-        if (!index) {
-            fix.setEntityMemberName("*");
-        }
-        fix
-            .setReceiver(new StreamFlattener())
-            .setReceiver(new StreamToTriples())
-            .setReceiver(tripleCount)
-            .setReceiver(new ObjectTemplate<Triple>(count ? "${s}\t ${o}" : "${s}"))
-            .setReceiver(getReceiver());
+        return tripleCount;
+    }
+
+    private TripleSort tripleSort() {
+        final TripleSort tripleSort = new TripleSort();
+        tripleSort.setNumeric(true);
+        tripleSort.setBy(Compare.OBJECT);
+        tripleSort.setOrder(Order.DECREASING);
+        return tripleSort;
     }
 
     @Override

diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixListPathsTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixListPathsTest.java
@@ -19,6 +19,7 @@
 import org.metafacture.framework.ObjectReceiver;
 
 import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
@@ -44,49 +45,74 @@ public MetafixListPathsTest() {
 
     @Test
     public void testShouldListPaths() {
-        processRecord();
-        verify("a.*\t 3");
+        verify(
+            "c.*\t 3",
+            "b.*\t 2",
+            "a\t 1");
     }
 
     @Test
     public void testShouldListPathsNoCount() {
         lister.setCount(false);
-        processRecord();
-        verify("a.*");
+        verify(
+            "a",
+            "b.*",
+            "c.*");
     }
 
     @Test
     public void testShouldListPathsUseIndex() {
         lister.setIndex(true);
-        processRecord();
-        verify("a.1\t 1");
-        verify("a.2\t 1");
-        verify("a.3\t 1");
+        verify(
+            "a\t 1",
+            "b.1\t 1",
+            "b.2\t 1",
+            "c.1\t 1",
+            "c.2\t 1",
+            "c.3\t 1");
     }
 
     @Test
     public void testShouldListPathsNoCountUseIndex() {
         lister.setCount(false);
         lister.setIndex(true);
-        processRecord();
-        verify("a.1");
-        verify("a.2");
-        verify("a.3");
+        verify(
+            "a",
+            "b.1",
+            "b.2",
+            "c.1",
+            "c.2",
+            "c.3");
+    }
+
+    @Test
+    public void testShouldListPathsSortedByFrequency() {
+        verify(
+            "c.*\t 3",
+            "b.*\t 2",
+            "a\t 1");
     }
 
     private void processRecord() {
         lister.setReceiver(receiver);
         lister.startRecord("");
-        lister.literal("a", "A");
-        lister.literal("a", "B");
-        lister.literal("a", "C");
+        lister.literal("a", "");
+        lister.literal("b", "");
+        lister.literal("b", "");
+        lister.literal("c", "");
+        lister.literal("c", "");
+        lister.literal("c", "");
         lister.endRecord();
         lister.closeStream();
     }
 
-    private void verify(final String result) throws MockitoAssertionError {
+    private void verify(final String... result) throws MockitoAssertionError {
+        processRecord();
         try {
-            Mockito.verify(receiver).process(result);
+            final InOrder ordered = Mockito.inOrder(receiver);
+            for (final String r : result) {
+                ordered.verify(receiver).process(r);
+            }
         }
         catch (final MockitoAssertionError e) {
             System.out.println(Mockito.mockingDetails(receiver).printInvocations());