Import from DB-Pedia

DBPedia exports all the entities as GZipped CSV files. Features:

• First line contains column names, second, third and forth has meta information we'll skip (look at "skipFrom": 1, "skipTo": 3in CSV transformer)
• The vertex class name is created automatically based on file name, so we can use the same file against any DBPedia file
• The Primary Key is the "URI" field where it has been created a UNIQUE index (look at "ORIENTDB" loader)
• The "merge" transformer is used to allow to reimport or update any file without generating duplicates

Configuration

{
  "config": {
    "verbose": true,
    "fileDirectory": "/temp/databases/dbpedia_csv/",
    "fileName": "Person.csv.gz"
  },
  "begin": [
   { "let": { "name": "$filePath",  "value": "$fileDirectory.append( $fileName )"} },
   { "let": { "name": "$className", "value": "$fileName.substring( 0, $fileName.indexOf(".") )"} }
  ],
  "source" : {
    "file": { "path": "$filePath", "lock" : true }
  },
  "extractor" : {
    "row": {}
  },
  "transformers" : [
   { "csv": { "separator": ",", "nullValue": "NULL", "skipFrom": 1, "skipTo": 3 } },
   { "merge": { "joinFieldName":"URI", "lookup":"V.URI" } },
   { "vertex": { "class": "$className"} }
  ],
  "loader" : {
    "orientdb": {
      "dbURL": "plocal:/temp/databases/dbpedia",
      "dbUser": "admin",
      "dbPassword": "admin",
      "dbAutoCreate": true,
      "tx": false,
      "batchCommit": 1000,
      "dbType": "graph",
      "indexes": [{"class":"V", "fields":["URI:string"], "type":"UNIQUE" }]
    }
  }
}