usage: pydat-populator [-h] [-c CONFIG] [--debug] [--debug-level DEBUG_LEVEL]
                    [-x EXCLUDE [EXCLUDE ...]] [-n INCLUDE [INCLUDE ...]]
                    [--ignore-field-prefixes [IGNORE_FIELD_PREFIXES [IGNORE_FIELD_PREFIXES ...]]]
                    [-e EXTENSION] [-v] [-s] [--pipelines PIPELINES]
                    [--shipper-threads SHIPPER_THREADS]
                    [--fetcher-threads FETCHER_THREADS]
                    [--bulk-ship-size BULK_SHIP_SIZE]
                    [--bulk-fetch-size BULK_FETCH_SIZE]
                    [-u [ES_URI [ES_URI ...]]] [--es-user ES_USER]
                    [--es-pass ES_PASSWORD] [--cacert ES_CA_CERT]
                    [--es-disable-sniffing] [-p ES_INDEX_PREFIX]
                    [--rollover-size ES_ROLLOVER_DOCS] [--ask-pass]
                    [-r | --config-template-only | --clear-interrupted-flag]
                    [-f INGEST_FILE | -d INGEST_DIRECTORY] [-D INGEST_DAY]
                    [-o COMMENT]

optional arguments:
-h, --help            show this help message and exit
-c CONFIG, --config CONFIG
                        location of configuration file for
                        environmentparameter configuration (example yaml file
                        in /backend)
--debug               Enables debug logging
--debug-level DEBUG_LEVEL
                        Debug logging level [0-3] (default: 1)
-x EXCLUDE [EXCLUDE ...], --exclude EXCLUDE [EXCLUDE ...]
                        list of keys to exclude if updating entry
-n INCLUDE [INCLUDE ...], --include INCLUDE [INCLUDE ...]
                        list of keys to include if updating entry (mutually
                        exclusive to -x)
--ignore-field-prefixes [IGNORE_FIELD_PREFIXES [IGNORE_FIELD_PREFIXES ...]]
                        list of fields (in whois data) to ignore when
                        extracting and inserting into ElasticSearch
-e EXTENSION, --extension EXTENSION
                        When scanning for CSV files only parse files with
                        given extension (default: csv)
-v, --verbose         Be verbose
-s, --stats           Print out Stats after running
-r, --redo            Attempt to re-import a failed import or import more
                        data, uses stored metadata from previous run
--config-template-only
                        Configure the ElasticSearch template and then exit
--clear-interrupted-flag
                        Clear the interrupted flag, forcefully (NOT
                        RECOMMENDED)
-f INGEST_FILE, --file INGEST_FILE
                        Input CSV file
-d INGEST_DIRECTORY, --directory INGEST_DIRECTORY
                        Directory to recursively search for CSV files --
                        mutually exclusive to '-f' option
-D INGEST_DAY, --ingest-day INGEST_DAY
                        Day to use for metadata, in the format 'YYYY-MM-dd',
                        e.g., '2021-01-01'. Defaults to todays date, use
                        'YYYY-MM-00' to indicate a quarterly ingest, e.g.,
                        2021-04-00
-o COMMENT, --comment COMMENT
                        Comment to store with metadata

Performance Options:
--pipelines PIPELINES
                        Number of pipelines (default: 2)
--shipper-threads SHIPPER_THREADS
                        How many threads per pipeline to spawn to send bulk ES
                        messages. The larger your cluster, the more you can
                        increase this, defaults to 1
--fetcher-threads FETCHER_THREADS
                        How many threads to spawn to search ES. The larger
                        your cluster, the more you can increase this, defaults
                        to 2
--bulk-ship-size BULK_SHIP_SIZE
                        Size of Bulk Elasticsearch Requests (default: 10)
--bulk-fetch-size BULK_FETCH_SIZE
                        Number of documents to search for at a time (default:
                        50), note that this will be multiplied by the number
                        of indices you have, e.g., if you have 10
                        pydat-<number> indices it results in a request for 500
                        documents

Elasticsearch Options:
-u [ES_URI [ES_URI ...]], --es-uri [ES_URI [ES_URI ...]]
                        Location(s) of ElasticSearch Server (e.g.,
                        foo.server.com:9200) Can take multiple endpoints
--es-user ES_USER     Username for ElasticSearch when Basic Auth is enabled
--es-pass ES_PASSWORD
                        Password for ElasticSearch when Basic Auth is enabled
--cacert ES_CA_CERT   Path to a CA Certicate bundle to enable https support
--es-disable-sniffing
                        Disable ES sniffing, useful when ssl
                        hostnameverification is not working properly
-p ES_INDEX_PREFIX, --index-prefix ES_INDEX_PREFIX
                        Index prefix to use in ElasticSearch (default: pydat)
--rollover-size ES_ROLLOVER_DOCS
                        Set the number of documents after which point a new
                        index should be created, defaults to 50 million, note
                        that this is fuzzy since the index count isn't
                        continuously updated, so should be reasonably below 2
                        billion per ES shard and should take your ES
                        configuration into consideration
--ask-pass            Prompt for ElasticSearch password

pydat-populator -u localhost:9200 -f ~/whois/data/1.csv -v -s -x Audit_auditUpdatedDate,updatedDate,standardRegUpdatedDate,expiresDate,standardRegExpiresDate

cd pydat/
docker build -t mitrecnd/pydat:5

version: '3'

services:
    pydat:
        image: mitrecnd/pydat:5
        volumes:
            - "./config.py:/opt/pydat/config.py:ro"
        ports:
            - 127.0.0.1:8888:8888

cd pydat/backend/
pip install ./

api/v1/metadata/

api/v1/metadata/<version>/

api/v1/domain/<domainName>/

api/v1/domain/<domainName>/latest/
api/v1/domain/<domainName>/<version>/
api/v1/domain/<domainName>/<version1>/<version2>/
api/v1/domain/<domainName>/diff/<version1>/<version2>/

api/v1/domains/<searchKey>/<searchValue>/

api/v1/domains/<searchKey>/<searchValue>/latest/
api/v1/domains/<searchKey>/<searchValue>/<version>/
api/v1/domains/<searchKey>/<searchValue>/<version1>/<version2>/

domainName

registrant_name
contactEmail
registrant_telephone

curl http://pydat.myorg.domain/ajax/domain/google.com/latest/

curl http://pydat.myorg.domain/ajax/domains/domainName/google.com/

api/v1/query

query - The query to search ES with
size - The number of elements to return (aka page size)
page - The page to return, combining this with size you can get the results in chunks
unique - Attempts to return the latest entry per domainName

api/v2/metadata

api/v2/metadata/<version>

api/v2/resolve/<domain>

api/v2/domains/diff [POST]

{

    domain: "mydomain.example",
    version1: 1,
    version2: 2
}

api/v2/domain [POST]

{

    value: "mydomain.example",
    version: 1, # Optional
    chunk_size: 50, # Optional
    offset: 0 # Optional
}

api/v2/query [POST]

{

    query: "myquery",
    chunk_size: 50, # Optional
    offset: 0, #Optional
    unique: false, # Optional
    sort_keys: [ # Optional
        "domainName",
        "registrant_name",
        "contactEmail",
        "standardRegCreatedDate",
        "registrant_telephone",
        "dataVersion",
        "_score",
    ]
}

api/v2/info

api/v2/settings