Merge pull request #4754 from ntisseyre/management_api

Schema Management APIs for Enhanced Debugging in JanusGraph

Author: ntisseyre

janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphCustomIdTest.java

@@ -16,6 +16,7 @@
 
 import com.google.common.base.Preconditions;
 import io.github.artsok.RepeatedIfExceptionsTest;
+import org.apache.commons.codec.DecoderException;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.T;
@@ -582,6 +583,38 @@ public void testWriteAndReadWithJanusGraphIoRegistryWithGraphson(@TempDir Path t
         testWritingAndReading(file.toFile());
     }
 
+    @Test
+    public void testGetVertexId() throws DecoderException {
+
+        open(true, true);
+
+        mgmt.makeVertexLabel("cat").make();
+        makeKey("id", Integer.class);
+        makeKey("name", String.class);
+
+        mgmt.commit();
+
+        Vertex v1 = tx.addVertex(T.id, 1000L, T.label, "cat");
+        v1.property("id", 1);
+        v1.property("name", "cat_1");
+        Object vertexId1 = v1.id();
+
+        Vertex v2 = tx.addVertex(T.id, "id_cat2", T.label, "cat");
+        v2.property("id", 2);
+        v2.property("name", "cat_2");
+        Object vertexId2 = v2.id();
+
+        tx.commit();
+
+        mgmt = graph.openManagement();
+
+        String hex1 = mgmt.getVertexKey(vertexId1);
+        String hex2 = mgmt.getVertexKey(vertexId2);
+
+        assertEquals(vertexId1, mgmt.getVertexId(hex1));
+        assertEquals(vertexId2, mgmt.getVertexId(hex2));
+    }
+
     private void testWritingAndReading(File f) {
         GraphTraversalSource g = graph.traversal();
         Vertex fromV = g.addV().property("name", f.getName()).property(T.id, "custom_id").next();

janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java

@@ -19,6 +19,7 @@
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Sets;
 import io.github.artsok.RepeatedIfExceptionsTest;
+import org.apache.commons.codec.DecoderException;
 import org.apache.tinkerpop.gremlin.process.traversal.P;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
@@ -96,6 +97,8 @@
 import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphStep;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexCountStrategy;
 import org.janusgraph.graphdb.transaction.StandardJanusGraphTx;
+import org.janusgraph.graphdb.types.CompositeIndexType;
+import org.janusgraph.graphdb.types.IndexField;
 import org.janusgraph.graphdb.types.MixedIndexType;
 import org.janusgraph.graphdb.types.ParameterType;
 import org.janusgraph.graphdb.types.StandardEdgeLabelMaker;
@@ -4346,4 +4349,37 @@ public void testMixedIndexAggregatedCountReturnsCorrectResult() {
             );
         }
     }
+
+    @Test
+    public void testGetIndexInfo() throws DecoderException {
+
+        clopen();
+
+        mgmt.makeVertexLabel("cat").make();
+
+        makeKey("id", Integer.class);
+        final PropertyKey nameKey = makeKey("name", String.class);
+
+        String indexName = "searchByName";
+        mgmt.buildIndex(indexName, Vertex.class)
+            .addKey(nameKey)
+            .buildCompositeIndex();
+        mgmt.commit();
+
+        mgmt = graph.openManagement();
+
+        Map<String, Object> fieldValues = new HashMap<>();
+        fieldValues.put("name", "someName");
+
+        String hexString = mgmt.getIndexKey(indexName, fieldValues);
+        CompositeIndexType indexType = mgmt.getIndexInfo(hexString);
+
+        IndexField[] indexFields = new IndexField[1];
+        indexFields[0] = IndexField.of(nameKey);
+
+        assertEquals(indexName, indexType.getName());
+        assertEquals(1, indexType.getFieldKeys().length);
+        assertEquals(nameKey, indexType.getFieldKeys()[0].getFieldKey());
+        assertEquals(0, indexType.getInlineFieldKeys().length);
+    }
 }

janusgraph-core/src/main/java/org/janusgraph/core/schema/JanusGraphManagement.java

@@ -14,6 +14,7 @@
 
 package org.janusgraph.core.schema;
 
+import org.apache.commons.codec.DecoderException;
 import org.apache.tinkerpop.gremlin.process.traversal.Order;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Element;
@@ -23,9 +24,11 @@
 import org.janusgraph.core.RelationType;
 import org.janusgraph.core.VertexLabel;
 import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture;
+import org.janusgraph.graphdb.types.CompositeIndexType;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -469,4 +472,33 @@ interface IndexBuilder {
      */
     String printIndexes();
 
+    /**
+     * Get vertexId for the given hex string
+     * @param hexString
+     * @return vertex Id
+     * @throws DecoderException
+     */
+    Object getVertexId(String hexString) throws DecoderException;
+
+    /**
+     * Get a hex string representing vertex id
+     * @param vertexId  vertex id
+     * @return a hex string representing bytes of vertex id
+     */
+    String getVertexKey(Object vertexId);
+
+    /**
+     * Get a hex string representing index key
+     * @param indexName  schema index name
+     * @param fieldValues index fields with values
+     * @return a hex string representing bytes of index key
+     */
+    String getIndexKey(String indexName, Map<String, Object> fieldValues);
+
+    /**
+     * Get index info from given hex string
+     * @param hexString
+     * @return composite index info
+     */
+    CompositeIndexType getIndexInfo(String hexString) throws DecoderException;
 }

janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java

@@ -525,6 +525,10 @@ public long getIndexIdFromKey(StaticBuffer key) {
         return IndexRecordUtil.getIndexIdFromKey(key, hashKeys, hashLength);
     }
 
+    public StaticBuffer getIndexKey(CompositeIndexType index, Map<String, Object> fieldValues) {
+        return IndexRecordUtil.getIndexKey(index, fieldValues, serializer, hashKeys, hashLength);
+    }
+
     public boolean isHashKeys() {
         return hashKeys;
     }

janusgraph-core/src/main/java/org/janusgraph/graphdb/database/management/ManagementSystem.java

@@ -16,6 +16,8 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -51,6 +53,7 @@
 import org.janusgraph.core.schema.SchemaStatus;
 import org.janusgraph.core.schema.VertexLabelMaker;
 import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.StaticBuffer;
 import org.janusgraph.diskstorage.configuration.BasicConfiguration;
 import org.janusgraph.diskstorage.configuration.ConfigOption;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
@@ -62,6 +65,7 @@
 import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics;
 import org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScanner;
 import org.janusgraph.diskstorage.log.Log;
+import org.janusgraph.diskstorage.util.StaticArrayBuffer;
 import org.janusgraph.graphdb.database.IndexSerializer;
 import org.janusgraph.graphdb.database.StandardJanusGraph;
 import org.janusgraph.graphdb.database.cache.SchemaCache;
@@ -545,6 +549,28 @@ public String printIndexes() {
         return this.printIndexes(true);
     }
 
+    @Override
+    public Object getVertexId(String hexString) throws DecoderException {
+        return this.graph.getIDManager().getKeyID(StaticArrayBuffer.of(Hex.decodeHex(hexString)));
+    }
+
+    @Override
+    public String getVertexKey(Object vertexId) {
+        return Hex.encodeHexString(graph.getIDManager().getKey(vertexId).asByteBuffer());
+    }
+
+    @Override
+    public String getIndexKey(String indexName, Map<String, Object> fieldValues) {
+        StaticBuffer staticBuffer = transaction.getCompositeIndexKey(indexName, fieldValues);
+        return Hex.encodeHexString(staticBuffer.asByteBuffer());
+    }
+
+    @Override
+    public CompositeIndexType getIndexInfo(String hexString) throws DecoderException {
+        StaticArrayBuffer indexKey = StaticArrayBuffer.of(Hex.decodeHex(hexString));
+        return transaction.getCompositeIndexInfo(indexKey);
+    }
+
     private String printIndexes(boolean calledDirectly) {
         StringBuilder sb = new StringBuilder();
         String pattern = "%-30s | %-11s | %-9s | %-14s | %-10s %10s |%n";

janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexRecordUtil.java

@@ -73,6 +73,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 
 import static org.janusgraph.util.encoding.LongEncoding.STRING_ENCODING_MARKER;
 
@@ -331,6 +332,19 @@ public static StaticBuffer getIndexKey(CompositeIndexType index, IndexRecordEntr
         return getIndexKey(index, IndexRecordUtil.getValues(record), serializer, hashKeys, hashLength);
     }
 
+    public static StaticBuffer getIndexKey(CompositeIndexType index, Map<String, Object> fieldValues, Serializer serializer, boolean hashKeys, HashingUtil.HashLength hashLength) {
+        Object[] values = Arrays.stream(index.getFieldKeys()).map(field -> {
+            String fieldName = field.getFieldKey().name();
+            if (fieldValues.containsKey(fieldName)) {
+                return fieldValues.get(fieldName);
+            } else {
+                throw new NoSuchElementException("Value for fieldName=" + fieldName + " is not provided for indexName=" + index.getName());
+            }
+        }).toArray();
+
+        return getIndexKey(index, values, serializer, hashKeys, hashLength);
+    }
+
     public static StaticBuffer getIndexKey(CompositeIndexType index, Object[] values, Serializer serializer, boolean hashKeys, HashingUtil.HashLength hashLength) {
         final DataOutput out = serializer.getDataOutput(8*DEFAULT_OBJECT_BYTELEN + 8);
         VariableLong.writePositive(out, index.longId());

janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java

@@ -49,9 +49,11 @@
 import org.janusgraph.diskstorage.BackendException;
 import org.janusgraph.diskstorage.BackendTransaction;
 import org.janusgraph.diskstorage.EntryList;
+import org.janusgraph.diskstorage.StaticBuffer;
 import org.janusgraph.diskstorage.indexing.IndexTransaction;
 import org.janusgraph.diskstorage.keycolumnvalue.MultiKeysQueryGroups;
 import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
+import org.janusgraph.diskstorage.util.StaticArrayBuffer;
 import org.janusgraph.diskstorage.util.time.TimestampProvider;
 import org.janusgraph.graphdb.database.EdgeSerializer;
 import org.janusgraph.graphdb.database.IndexSerializer;
@@ -1243,6 +1245,34 @@ public VertexLabel getOrCreateVertexLabel(String name) {
         return vertexLabel;
     }
 
+    public StaticBuffer getCompositeIndexKey(String indexName, Map<String, Object> fieldValues) {
+        JanusGraphSchemaVertex schemaVertex = getSchemaVertex(JanusGraphSchemaCategory.GRAPHINDEX.getSchemaName(indexName));
+        Preconditions.checkNotNull(schemaVertex, "Index with name [" + indexName + "] was not found");
+        IndexType indexType = schemaVertex.asIndexType();
+        if (indexType instanceof CompositeIndexType) {
+            CompositeIndexType compositeIndex = (CompositeIndexType) indexType;
+            StaticBuffer indexKey = indexSerializer.getIndexKey(compositeIndex, fieldValues);
+            return indexKey;
+        } else {
+            throw new IllegalArgumentException("Index with name [" + indexName + "] is not a composite index");
+        }
+    }
+
+    public CompositeIndexType getCompositeIndexInfo(StaticArrayBuffer indexKey) {
+        long schemaVertexId = indexSerializer.getIndexIdFromKey(indexKey);
+        InternalVertex typeVertex = vertexCache.get(schemaVertexId, existingVertexRetriever);
+
+        Preconditions.checkNotNull(typeVertex, "Index with key [" + indexKey + "] was not found");
+        JanusGraphSchemaVertex schemaVertex = (JanusGraphSchemaVertex) typeVertex;
+
+        IndexType indexType = schemaVertex.asIndexType();
+        if (indexType instanceof CompositeIndexType) {
+            return (CompositeIndexType) indexType;
+        } else {
+            throw new IllegalArgumentException("Index with key [" + indexKey + "] is not a composite index");
+        }
+    }
+
     @Override
     public VertexLabelMaker makeVertexLabel(String name) {
         StandardVertexLabelMaker maker = new StandardVertexLabelMaker(this);