Skip to main content

Adding custom fields to a default index

This guide walks you through converting a default index to a custom one in order to add new custom fields to an existing default index that was created after the creation of the tenant. In this way, you can extend the default index fields with new fields that may be needed to represent updates on your specific document structure and search requirements.

By the end, you'll have managed to add a custom field to an existing default index.

Prerequisites​

Before adding a custom field to an existing default index, ensure you have:

  1. Access to the Zeta Alpha Platform UI with appropriate permissions
  2. An existing tenant (see Create a Tenant if you haven't created one) with a default index (i.e. an index that has no document_fields_configuration configured)

Step 1: Add a document_fields_configuration replicating the default index built-in configuration​

  1. In the Zeta Alpha Platform UI, open the index settings of the tenant.
  2. Check the document_fields_configuration of the index and verify that it is not configured. Note that if the document_fields_configuration is configured, it means that the index is not a default index anymore and you can add new custom fields by just adding the field based on the documentation of the document_fields_configuration in Index Reference.
  3. If the index is indeed a default index, then add the following document_fields_configuration to the existing index:
"document_fields_configuration": [
    {
      "name": "DCMI.title",
      "type": "string",
      "alias": "metadata.DCMI.title",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": true
      },
      "analyzer_options": {
        "analyzer": "english_nostem_analyzer"
      }
    },
    {
      "name": "metadata.DCMI.title",
      "type": "string",
      "alias": "metadata.DCMI.title",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": true
      },
      "analyzer_options": {
        "analyzer": "english_nostem_analyzer"
      }
    },
    {
      "name": "DCMI.abstract",
      "type": "string",
      "alias": "metadata.DCMI.abstract",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": false,
        "is_used_in_search": true
      },
      "analyzer_options": {
        "analyzer": "english_nostem_analyzer"
      }
    },
    {
      "name": "metadata.DCMI.abstract",
      "type": "string",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": false,
        "is_used_in_search": true
      },
      "analyzer_options": {
        "analyzer": "english_nostem_analyzer"
      }
    },
    {
      "name": "metadata.DCMI.language",
      "type": "string",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": true,
        "is_filter_field": true,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.language",
      "type": "string",
      "alias": "metadata.DCMI.language",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": true,
        "is_filter_field": true,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "metadata.DCMI.valid",
      "type": "date",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.valid",
      "type": "date",
      "alias": "metadata.DCMI.valid",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.modified",
      "type": "date",
      "alias": "metadata.DCMI.modified",
      "search_options": {
        "is_sort_field": true,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "metadata.DCMI.modified",
      "type": "date",
      "search_options": {
        "is_sort_field": true,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.source",
      "type": "string",
      "alias": "metadata.DCMI.source",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": true,
        "is_filter_field": true,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "metadata.DCMI.source",
      "type": "string",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": true,
        "is_filter_field": true,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.identifier",
      "type": "string",
      "alias": "metadata.DCMI.identifier",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": true,
        "is_returned_in_search_results": false,
        "is_used_in_search": false
      }
    },
    {
      "name": "metadata.DCMI.identifier",
      "type": "string",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": true,
        "is_returned_in_search_results": false,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.created",
      "type": "date",
      "alias": "metadata.DCMI.created",
      "search_options": {
        "is_sort_field": true,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "metadata.DCMI.created",
      "type": "date",
      "search_options": {
        "is_sort_field": true,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      }
    },
    {
      "name": "DCMI.resources",
      "type": "nested",
      "alias": "metadata.DCMI.resources",
      "nested_fields": [
        {
          "name": "resource_type",
          "type": "string",
          "search_options": {
            "is_sort_field": false,
            "is_facet_field": false,
            "is_filter_field": false,
            "is_returned_in_search_results": true,
            "is_used_in_search": false
          },
          "ranking_options": {}
        },
        {
          "name": "resource_value",
          "type": "string",
          "search_options": {
            "is_sort_field": false,
            "is_facet_field": false,
            "is_filter_field": false,
            "is_returned_in_search_results": true,
            "is_used_in_search": false
          },
          "ranking_options": {}
        }
      ]
    },
    {
      "name": "metadata.DCMI.resources",
      "type": "nested",
      "nested_fields": [
        {
          "name": "resource_type",
          "type": "string",
          "search_options": {
            "is_sort_field": false,
            "is_facet_field": false,
            "is_filter_field": false,
            "is_returned_in_search_results": true,
            "is_used_in_search": false
          },
          "ranking_options": {}
        },
        {
          "name": "resource_value",
          "type": "string",
          "search_options": {
            "is_sort_field": false,
            "is_facet_field": false,
            "is_filter_field": false,
            "is_returned_in_search_results": true,
            "is_used_in_search": false
          },
          "ranking_options": {}
        }
      ]
    },
    {
      "name": "DCMI.creator",
      "type": "nested",
      "alias": "metadata.DCMI.creator",
      "search_options": {
        "is_sort_field": false,
        "is_facet_field": false,
        "is_filter_field": false,
        "is_returned_in_search_results": true,
        "is_used_in_search": false
      },
      "nested_fields": [
        {
          "name": "full_name",
          "type": "string",
          "search_options": {
            "is_sort_field": false,
            "is_facet_field": false,
            "is_filter_field": false,
            "is_returned_in_search_results": true,
            "is_used_in_search": false
          },
          "ranking_options": {}
        },
        {
          "name": "Ontology_ID",
          "type": "string",
          "search_options": {
            "is_sort_field": false,
            "is_facet_field": false,
            "is_filter_field": false,
            "is_returned_in_search_results": true,
            "is_used_in_search": false
          },
          "ranking_options": {}
        }
      ]
    },
    {
      "name": "metadata.DCMI.creator",
      "type": "nested",
      "search_options": {
        "is_returned_in_search_results": true
      },
      "nested_fields": [
        {
          "name": "Ontology_ID",
          "type": "string",
          "search_options": {
            "is_facet_field": false,
            "is_filter_field": true,
            "is_returned_in_search_results": false,
            "is_used_in_search": false
          }
        },
        {
          "name": "full_name",
          "type": "string",
          "search_options": {
            "is_filter_field": true,
            "is_returned_in_search_results": true,
            "is_used_in_search": true
          }
        }
      ]
    }
  ]

Verify that the status of the index is still operational.

Step 2: Update user documents content source​

Once the index document_fields_configuration has been set up as described in the previous step, it is also necessary to update the field_mappings of the built-in user-documents-ingestion content source, if available. The mapping should now point to the metadata.DCMI. fields that we added in the previous step in the index configuration instead of the DCMI. fields. Specifically, the following field_mappings should be configured for the user-documents-ingestion content source:

"field_mappings": [
  {
    "content_source_field_name": "date",
    "index_field_name": "metadata.DCMI.created"
  },
  {
    "content_source_field_name": "source",
    "index_field_name": "metadata.DCMI.source"
  },
  {
    "content_source_field_name": "description",
    "index_field_name": "metadata.DCMI.abstract"
  },
  {
    "content_source_field_name": "title",
    "index_field_name": "metadata.DCMI.title"
  },
  {
    "content_source_field_name": "identifier",
    "index_field_name": "metadata.DCMI.identifier"
  },
  {
    "content_source_field_name": "authors",
    "index_field_name": "metadata.DCMI.creator",
    "inner_field_mappings": [
      {
        "content_source_field_name": ".",
        "index_field_name": "full_name"
      }
    ]
  },
  {
    "content_source_field_name": "metadata.DCMI.resources",
    "index_field_name": "resources",
    "inner_field_mappings": [
      {
        "content_source_field_name": "resource_type",
        "index_field_name": "resource_type"
      },
      {
        "content_source_field_name": "resource_value",
        "index_field_name": "resource_value"
      }
    ]
  }
]

Step 3: Add a new custom field​

Once you have configured the document_fields_configuration with the default built-in fields in Step 1, you can now add your new custom fields by extending the list of fields under document_fields_configuration following the documentation of the document_fields_configuration in Index Reference.

For example, if you want to add a field publisher in your index you would need to append the following field in the document_fields_configuration list:

{
  "name": "publisher",
  "type": "string",
  "search_options": {
    "is_sort_field": false,
    "is_facet_field": false,
    "is_filter_field": true,
    "is_returned_in_search_results": true,
    "is_used_in_search": false
  }
}

Verify that the status of the index is still operational.

Conclusion​

Congratulations! You've successfully added a custom field to a default index. You can now push data to this new custom field, through the field_mappings of the built-in content source configurations or through the ingestion API by passing the custom field inside the custom_metadata of the document request similarly to the Adding a document quickstart guide

If you have any questions or encounter any issues, please don't hesitate to reach out to our support team.