#!/home/conda/feedstock_root/build_artifacts/dqsegdb_1771236899807/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin/python
#
# Copyright (C) 2014-2020 Syracuse University, European Gravitational Observatory, and Christopher Newport University.
# Written by Ryan Fisher and Gary Hemming. See the NOTICE file distributed with this work for additional information
# regarding copyright ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Adapted from ligolw_segment_query:
# Copyright (C) 2009  Larne Pekowsky, Ping Wei

#
# =============================================================================
#
#                                   Preamble
#
# =============================================================================
#


"""
This provides the means to answer several questions posed against either the
segment database or a collection of DMT XML files.  Output should match
exactly the format returned by S6 style segment database tools.

  * What DQ flags exist in the database? ligolw_segment_query --show-types
  * When was a given DQ flag defined? ligolw_segment_query --query-types
  * When was a given flag active? ligolw_segment_query --query-segments
* Example: ligolw_segment_query_dqsegdb --segment-url=https://segments.ligo.org --query-segments \
--gps-start-time 987654321 --gps-end-time 987654333 --include-segments="H1:DMT-SCIENCE:1" -o example.xml
"""

import json
import os
import pwd
import sys
import tempfile
import warnings

import igwn_segments as segments

from optparse import OptionParser
from urllib.parse import urlparse
from urllib.request import urlopen

from igwn_ligolw import dbtables
from igwn_ligolw import ligolw
from igwn_ligolw import lsctables
from igwn_ligolw import utils
from igwn_ligolw import types as ligolwtypes
from igwn_ligolw.ligolw import Table
from igwn_ligolw.utils import ligolw_sqlite

from dqsegdb import apicalls
from dqsegdb import clientutils
from dqsegdb import urifunctions
from dqsegdb._version import get_versions
from dqsegdb.segmentdb import query_engine
from dqsegdb.segmentdb import segmentdb_utils

debug = False
#debug = True

PROGRAM_NAME = sys.argv[0].replace('./', '')
PROGRAM_PID = os.getpid()
try:
    USER_NAME = os.getlogin()
except:
    USER_NAME = pwd.getpwuid(os.getuid())[0]


__author__ = "Ryan Fisher <ryan.fisher@ligo.org>, Larne Pekowsky <lppekows@physics.syr.edu>, Ping Wei <piwei@syr.edu>"
#__version__ = "git id %s" % git_version.id
__version__ = get_versions()['version']
__id__ = get_versions()['full-revisionid']
__date__ = get_versions()['date']
__verbose_msg__ = "version: " + get_versions()['version'] + "\nfull-revisionid: " + get_versions()['full-revisionid'] \
    + "\ndirty: " + str(get_versions()['dirty']) + "\nerror: " + str(get_versions()['error']) + "\ndate: " \
    + get_versions()['date']
__formatted_date__ = __date__[0:10] + " " + __date__[11:19] + " +0000"
#__formatted_date__ = __date__[0:10] + " " + __date__[11:19] + " " + __date__[19:25]
# it would be nice to fix this [above commented-out line] so that the time isn't off by a few hours


#
# =============================================================================
#
#                                 Command Line
#
# =============================================================================
#


def parse_command_line():
    """
    Parse the command line, return an options object
    """

    parser = OptionParser(
        version="Name: %prog\n" + __verbose_msg__,
        usage="%prog [ --version | --ping | --show-types | --query-types | --query-segments ]  [ --segment "
              "| --database | --dmt-files ] options ",
        description="Performs a number of queries against either a set of DMT files or a segment database"
    )

    # Major modes
    parser.add_option("-p", "--ping", action="store_true", help="Ping the target server")
    parser.add_option("-y", "--show-types", action="store_true", help="Returns an xml table containing segment type "
                      "information: ifos, name, version, segment_definer.comment, segment_summary.start_time, "
                      "segment_summary.end_time, segment_summary.comment")
    parser.add_option("-u", "--query-types", action="store_true", help="Returns a ligolw document whose "
                      "segment_definer table includes all segment types defined in the given period and included by "
                      "include-segments and whose segment_summary table indicates the times for which those segments "
                      "are defined. Warning: Currently only support unversioned include-segments strings!")
    parser.add_option("-q", "--query-segments", action="store_true", help="Returns a ligolw document whose segment "
                      "table contains the times included by the include-segments flag and excluded by "
                      "exclude-segments")

    # Time options
    parser.add_option("-s", "--gps-start-time", metavar="gps_start_time", help="Start of GPS time range")
    parser.add_option("-e", "--gps-end-time", metavar="gps_end_time", help="End of GPS time range")

    # Data location options
    parser.add_option("-t", "--segment-url", metavar="segment_url", help="Segment URL. Users have to specify either "
                      "'https://' for a secure connection or 'http://' for an insecure connection to the segment "
                      "database URL. For example, '--segment-url=https://segments.ligo.org'. No need to specify port "
                      "number. ")
    parser.add_option("-d", "--database", metavar="use_database", action="store_true", help="Use database specified by "
                      "environment variable DEFAULT_SEGMENT_SERVER. For example, "
                      "'DEFAULT_SEGMENT_SERVER=https://segments.ligo.org'")
    parser.add_option("-f", "--dmt-files", metavar="use_files", action="store_true", help="Use files in directory "
                      "specified by environment variable ONLINEDQ, for example, 'ONLINEDQ=file:///path_to_dmt'. "
                      "'file://' is the prefix, the actual directory to DMT xml files starts with '/'.")

    # Other options
    parser.add_option("-a", "--include-segments", metavar="include_segments", help="This option expects a "
                      "comma-separated list of a colon-separated sublist of interferometer, segment type, and version. "
                      "The union of segments from all types and versions specified is returned. Use --show-types to "
                      "see what types are available.   For example: --include-segment-types "
                      "H1:DMT-SCIENCE:1,H1:DMT-INJECTION:2 will return the segments for which H1 is in either SCIENCE "
                      "version 1 or INJECTION version 2 mode. If version information is not provided, the union of the "
                      "segments of the latest version of requested segment type(s) will be returned.")

    parser.add_option("-b", "--exclude-segments", metavar="exclude_segments", help="This option has to be used in "
                      "conjunction with --include-segment-types and --query-segments.  --exclude-segment-types "
                      "subtracts the union of unwanted segments from the specified types from the results of "
                      "--include-segment-types. If version information is not provided, --exclude-segment-types "
                      "subtracts the union of segments from the latest version of the specified segment types. For "
                      "example, --include-segment-types H1:DMT-SCIENCE:1,H1:DMT-INJECTION:2 --exclude-segment-types "
                      "H1:DMT-WIND:1,H1:DMT-NOT_LOCKED:2,H2:DMT-NOT_LOCKED:2 will subtract the union of segments which "
                      "H1 is in version 1 WIND and H1,H2 is version 2 NOT_LOCKED from the result of "
                      "--include-segment-types H1:DMT-SCIENCE:1,H1:DMT-INJECTION:2")

    parser.add_option("-S", "--strict-off", metavar="use_strict", action="store_true", help="The default behavior is "
                      "to truncate segments so that returned segments are entirely in the interval [gps-start-time, "
                      "gps-end-time).  However if this option is given, the entire non-truncated segment is returned "
                      "if any part of it overlaps the interval.")

    parser.add_option("-n", "--result-name", metavar="result_name", default="RESULT", help="Name for result segment "
                      "definer (default = RESULT)")

    parser.add_option("-o", "--output-file", metavar="output_file", help="File to which output should be written.  "
                      "Defaults to stdout.")
    #parser.add_option("-v", "--debug", metavar="debug", help= "Turn on debugging.")

    options, others = parser.parse_args()

    # Make sure we have exactly one thing to do
    count = 0
    for arg in [options.ping, options.query_types, options.show_types, options.query_segments]:
        if arg:
            count += 1

    if count != 1:
        raise ValueError("Exactly one of [ --ping | --show-types | --query-types | --query-segments ] must be provided")

    # Make sure we have required arguments
    database_location = None
    file_location = None

    # Make sure we know who to contact for data
    if options.segment_url:
        if options.segment_url.startswith('http'):
            database_location = options.segment_url
        elif options.segment_url.startswith('file:'):
            file_location = options.segment_url[len('file://'):]
        else:
            tmp_dir = tempfile.mkdtemp()

            # Grab the part of the name after the last slash
            pos = options.segment_url[::-1].find('/')
            fname = (pos > -1) and options.segment_url[-1 * pos:] or "dmt.xml"

            inurl = urlopen(options.segment_url)
            outfile = open(tmp_dir + "/" + fname, 'w')
            for l in inurl:   # FIXME flake8 warns: "E741 ambiguous variable name 'l'"
                print(l, file=outfile)

            inurl.close()
            outfile.close()
            file_location = tmp_dir
    elif options.database:
        try:
            database_location = os.environ['DEFAULT_SEGMENT_SERVER']
        except KeyError:
            raise ValueError("--database specified but DEFAULT_SEGMENT_SERVER not set")
    elif options.dmt_files:
        if 'ONLINEDQ' not in os.environ:
            raise ValueError("--dmt-files specified but ONLINEDQ not set")
        tmp = os.environ['ONLINEDQ']
        if tmp.startswith('file://'):
            tmp = tmp[len('file://'):]
        file_location = tmp
    else:
        raise ValueError("One of [ --segment-url | --database | --dmt-files ] must be provided")

    # Unless we're pinging, make sure we have start and end times
    if options.ping:
        if not database_location:
            raise ValueError("--ping requires [ --segment-url https://... | --database ]")
    else:
        if not options.gps_start_time:
            raise ValueError("missing required argument --gps-start-time")

        if not options.gps_end_time:
            raise ValueError("missing required argument --gps-end-time")

        if not options.show_types and not options.include_segments:
            raise ValueError("missing required argument --include-segments")

    return options, database_location, file_location


#
# =============================================================================
#
#                                 General utilities
#
# =============================================================================
#


def seg_spec_to_sql(spec):
    """Given a string of the form ifo:name:version, ifo:name:* or ifo:name
    constructs a SQL clause to restrict a search to that segment definer"""

    parts = spec.split(':')
    sql = "(segment_definer.ifos = '%s'" % parts[0]

    if len(parts) > 1 and parts[1] != '*':
        sql += " AND segment_definer.name = '%s'" % parts[1]
        if len(parts) > 2 and parts[2] != '*':
            sql += " AND segment_definer.version = %s" % parts[2]

    sql += ')'

    return sql


#
# The results of show-types is a join against segment_definer and segment
# summary, and so does not fit into an existing table type.  So here we
# define a new type so that the ligolw routines can generate the XML
#
class ShowTypesResultTable(Table):
    tableName = "show_types_result:table"

    validcolumns = {
        "ifos": "lstring",
        "name": "lstring",
        "version": "int_4s",
        "segment_definer_comment": "lstring",
        "segment_summary_start_time": "int_4s",
        "segment_summary_end_time": "int_4s",
        "segment_summary_comment": "lstring"
    }


class ShowTypesResult(object):
    __slots__ = list(ShowTypesResultTable.validcolumns.keys())

    def get_pyvalue(self):
        if self.value is None:
            return None
        return ligolwtypes.ToPyType[self.type or "lstring"](self.value)


ShowTypesResultTable.RowType = ShowTypesResult


#
# =============================================================================
#
#                          Methods that implement major modes
#
# =============================================================================
#
def run_show_types_dqsegdb(doc, protocol, server, gps_start_time, gps_end_time,
                           included_segments_string, excluded_segments_string):
    resulttable = lsctables.New(ShowTypesResultTable)
    doc.childNodes[0].appendChild(resulttable)

    #sql = """SELECT segment_definer.ifos, segment_definer.name, segment_definer.version,
    #             (CASE WHEN segment_definer.comment IS NULL THEN '-' WHEN segment_definer.comment IS NOT NULL THEN segment_definer.comment END),
    #             segment_summary.start_time, segment_summary.end_time,
    #             (CASE WHEN segment_summary.comment IS NULL THEN '-' WHEN segment_summary.comment IS NOT NULL THEN segment_summary.comment END)
    #      FROM  segment_definer, segment_summary
    #      WHERE segment_definer.segment_def_id = segment_summary.segment_def_id
    #      AND   NOT (segment_summary.start_time > %d OR %d > segment_summary.end_time)
    #      """ % (gps_end_time, gps_start_time)

    #rows = engine.query(sql)
    # Note:
    #Type and example of first row in rows:
    #    <type 'tuple'>
    #    ('H1          ', 'ODC-PSL_FSS_RFPD_LT_TH', 1, 'RPFD check, when above threshold the segment will be off', \
    #        1072880640, 1072880656, '-')

    seg_dict = {}

    includeSegments = True
    # Note: 4th argument is verbosity of call, prints url, etc.
    result, queryurl = apicalls.reportKnown(protocol, server, includeSegments, False, gps_start_time, gps_end_time)
    ## Fix (future change)!!! decide if we should still cut on included or excluded segments:  current decision is
    #    to not cut
    rows = apicalls.parseKnown(result)

    for row in rows:
        ifos, name, version, segment_definer_comment, \
            segment_summary_start_time, segment_summary_end_time, segment_summary_comment = row
        key = (ifos, name, version, segment_definer_comment, segment_summary_comment)
        if key not in seg_dict:
            seg_dict[key] = []

        seg_dict[key].append(segments.segment(segment_summary_start_time, segment_summary_end_time))

    for key, value in list(seg_dict.items()):
        segmentlist = segments.segmentlist(value)
        segmentlist.coalesce()

        for segment in segmentlist:
            result = ShowTypesResult()
            result.ifos, result.name, result.version, \
                result.segment_definer_comment, result.segment_summary_comment = key
            result.segment_summary_start_time, result.segment_summary_end_time = segment
            result.ifos = result.ifos.strip()

            resulttable.append(result)

    #engine.close()


def run_query_types_dqsegdb(doc, process_id, protocol, server, gps_start_time, gps_end_time, include_segments):
    # def run_query_types(doc, proc_id, connection, engine, gps_start_time, gps_end_time, included_segments):
    warnings.warn("Warning: Currently only support unversioned include-segments strings!  "
                  "Versioned queries will return incorrect results.")
    query_segment = segments.segmentlist([segments.segment(gps_start_time, gps_end_time)])

    #sql = """SELECT segment_definer.ifos, segment_definer.name,segment_definer.version,
    #       (CASE WHEN segment_definer.comment IS NULL THEN '-' WHEN segment_definer.comment IS NOT NULL THEN segment_definer.comment END),
    #       segment_summary.start_time, segment_summary.end_time,
    #       (CASE WHEN segment_summary.comment IS NULL THEN '-' WHEN segment_summary.comment IS NOT NULL THEN segment_summary.comment END)
    #FROM segment_definer, segment_summary
    #WHERE segment_definer.segment_def_id = segment_summary.segment_def_id
    #AND NOT(%d > segment_summary.end_time OR segment_summary.start_time > %d)
    #""" % (gps_start_time, gps_end_time)

    #type_clauses = map(seg_spec_to_sql, included_segments.split(','))

    #dqsegdb:  I need to allow for cascaded queries in this format:
    #  This just means that I need to filter my results to include things that match the query, with the option to
    #    wildcard the version number

    segments_to_query = []  # This is a list of 3 element lists ifo,name,version when completed

    for included in include_segments.split(','):
        spec = included.split(':')

        if len(spec) < 2 or len(spec) > 3:
            raise ValueError("Included segements must be of the form ifo:name:version or ifo:name:*")
            #sys.exit(1)
        ifo = spec[0]
        name = spec[1]        # segdefs are actually segment summaries and need to look like this:
        if len(spec) == 3 and spec[2] != '*':
            # This is a versioned query
            try:
                version = int(spec[2])
            except ValueError:
                raise ValueError("Bad version specification, please provide a valid integer.")
                #raise
            if version < 1:
                raise ValueError("Segment version numbers must be greater than zero")
                sys.exit(1)
            # Ok, we got a properly versioned query item, add it to the list:
            segments_to_query.append(spec)
        else:
            versionQueryURL = urifunctions.constructVersionQueryURL(protocol, server, ifo, name)
            if debug:
                print(versionQueryURL)
            versionResult = urifunctions.getDataUrllib2(versionQueryURL)
            versionData = json.loads(versionResult)  # JSON is nice... :)
            # Construct urls and issue queries for the multiple versions and dump the results to disk locally for
            #   careful provenance
            jsonResults = []
            #urlList=versionData['resource_type']
            version_list = versionData['version']
            for version in version_list:
                segments_to_query.append([ifo, name, version])

    segment_types = {}
    ### Fix!!! Make a call to /report/known?s=t1&e=t2
    ### then parse the JSON result exactly as is done below
    includeSegments = True
    # Note: 4th argument is verbosity of call, prints url, etc.
    result, queryurl = apicalls.reportKnown(protocol, server, includeSegments, False, gps_start_time, gps_end_time)
    ## Fix (future change)!!! decide if we should still cut on included or excluded segments:  current decision
    #    is to not cut
    rows = apicalls.parseKnown(result)

    for row in rows:
        ifo, name, version, segment_definer_comment, segment_summary_start_time, \
            segment_summary_end_time, segment_summary_comment = row
        if [ifo, name, version] in segments_to_query:
            key = (ifo, name, version, segment_definer_comment, segment_summary_comment)
            if key not in segment_types:
                segment_types[key] = segments.segmentlist([])
            segment_types[key] |= segments.segmentlist([segments.segment(segment_summary_start_time,
                                                                         segment_summary_end_time)])

    # Create segment definer and segment_summary tables
    seg_def_table = lsctables.New(lsctables.SegmentDefTable, columns=[
        "process:process_id", "segment_def_id", "ifos", "name", "version", "comment"])
    doc.childNodes[0].appendChild(seg_def_table)

    seg_sum_table = lsctables.New(lsctables.SegmentSumTable, columns=[
        "process:process_id", "segment_sum_id", "start_time", "start_time_ns", "end_time", "end_time_ns", "comment",
        "segment_definer:segment_def_id"])

    doc.childNodes[0].appendChild(seg_sum_table)
    if debug:
        print(segment_types)

    for key in segment_types:
        # Make sure the intervals fall within the query window and coalesce
        segment_types[key].coalesce()
        segment_types[key] &= query_segment

        seg_def_id                     = seg_def_table.get_next_id()   # noqa: E221
        segment_definer                = lsctables.SegmentDef()        # noqa: E221
        segment_definer.process_id     = process_id                    # noqa: E221
        segment_definer.segment_def_id = seg_def_id                    # noqa: E221
        segment_definer.ifos           = key[0]                        # noqa: E221
        segment_definer.name           = key[1]                        # noqa: E221
        segment_definer.version        = key[2]                        # noqa: E221
        segment_definer.comment        = key[3]                        # noqa: E221

        seg_def_table.append(segment_definer)

        # add each segment summary to the segment_summary_table

        for seg in segment_types[key]:
            segment_sum                = lsctables.SegmentSum()        # noqa: E221
            segment_sum.comment        = key[4]                        # noqa: E221
            segment_sum.process_id     = process_id                    # noqa: E221
            segment_sum.segment_def_id = seg_def_id                    # noqa: E221
            segment_sum.segment_sum_id = seg_sum_table.get_next_id()   # noqa: E221
            segment_sum.start_time     = seg[0]                        # noqa: E221
            segment_sum.start_time_ns  = 0                             # noqa: E221
            segment_sum.end_time       = seg[1]                        # noqa: E221
            segment_sum.end_time_ns    = 0                             # noqa: E221

            seg_sum_table.append(segment_sum)


def run_query_segments_dqsegdb(
        doc, process_id, protocol, server, gps_start_time, gps_end_time,
        include_segments, exclude_segments, result_name):
    include_list_string = 'active,known,metadata'
    segdefs = []
    startTime = gps_start_time
    endTime = gps_end_time

    ## How to tackle versionless and include/exclude at same time:
    # a. first do any versionless queries, then put result into included_JSON_results or excluded_JSON_results
    #      and put the partial (versioned) answers into a different list, called included_intermediate_JSON

    # Let's try a.:

    # Ok the included_JSON_results and excluded_JSON_results will be handed to
    # the clientutils.calculate_combined_result function to generate the
    # final segments for the combined result
    # *_intermediate_JSON will contain the by-products of versionless queries
    # where by-products mean the versioned results that are used to make the
    # combined versionless result
    included_JSON_results = []
    included_intermediate_JSON = []
    excluded_JSON_results = []
    excluded_intermediate_JSON = []
    #segment_summaries = segments.segmentlist([])
    segment_summaries = []

    for included in include_segments.split(','):
        spec = included.split(':')

        if len(spec) < 2 or len(spec) > 3:
            print("Included segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr)
            sys.exit(1)

        ifo = spec[0]
        name = spec[1]
        # segdefs are actually segment summaries and need to look like this:
        # [('H1', 'DCH-TEST', 2, 1000000142, 1000000146, 0, 0), ('H1', 'DCH-TEST', 1 , 999999999, 1000000020, 0, 0), \
        #  ('H1', 'DCH-TEST', 1, 1000000021, 1000000121, 0, 0), ('H1', 'DCH-TEST', 1, 1000000136, 1000000139, 0, 0), \
        #  ('H1', 'DCH-TEST', 1, 1000000140, 1000000142, 0, 0)]
        # Of particular note: single version queries just use startTime and endTime, but versionless return the start
        # and end time of the segment summaries that affect the final versionless result
        # This means that I need the versionless query above to return that additional information

        if len(spec) == 3 and spec[2] != '*':
            # This is a versioned query
            try:
                version = int(spec[2])
            except ValueError:
                raise ValueError("Bad version specification, please provide a valid integer.")
            if version < 1:
                print("Segment version numbers must be greater than zero", file=sys.stderr)
                sys.exit(1)
            # call to single version flag query goes here!
            # result is already processed into a python dict using json.loads :
            json_result, queryurl = apicalls.dqsegdbQueryTimesCompatible(protocol, server, ifo, name, version,
                                                                         include_list_string, startTime, endTime)
            if debug:
                print(json_result)
                print(queryurl)
            included_JSON_results.append(json_result)
            segdefs.append((ifo, name, version, startTime, endTime, 0, 0))
            known_segments = segments.segmentlist([segments.segment(x[0], x[1]) for x in json_result['known']])
            segment_summaries.append(known_segments & segments.segmentlist([segments.segment(startTime, endTime)]))
        else:
            version = '*'
            # Trigger versionless call to return a JSON result here
            json_result, versioned_JSON, affected_results = \
                apicalls.dqsegdbCascadedQuery(protocol, server, ifo, name, include_list_string, startTime, endTime)
            if debug:
                print("Cascaded query call results:")
                print(json_result)
                print("Versioned intermediate results")
                print(versioned_JSON)
                print("Affected results segments")
                print(affected_results)
            # affected_results contains the segments of time that affected the final result for each version used
            #   in the calculation
            # This replicates the S6 behavior:
            included_JSON_results.append(json_result)
            for i_result in versioned_JSON:
                # Note: doesn't run if versioned_JSON is empty due to flag not existing
                included_intermediate_JSON.append(i_result)
                for seg in affected_results[i_result['version']]:
                    segdefs.append((ifo, name, i_result['version'], seg[0], seg[1], 0, 0))
                    # Ok, so here the segment_summaries are paired with the affected_results from above:  the
                    #    intersection of the affected results segments with each segment_summary for each version...
                    # First need to convert the known list to a segmentlist objet:
                    # Note that it appears I just need one segment summary that matches each segment definer above.
                    known_segments = segments.segmentlist([segments.segment(x[0], x[1]) for x in i_result['known']])
                    segment_summaries.append(known_segments & segments.segmentlist([segments.segment(seg[0], seg[1])]))

    # And we could write out everything we found
    if debug:
        print("Information on segdefs and segment_summaries: lengths, then first 10 entries")
        print(len(segdefs))
        print(len(segment_summaries))
        if len(segdefs) > 10:
            print(segdefs[:10])
        else:
            print(segdefs)
        if len(segment_summaries) > 10:
            print("Lots of summaries:")
            print(segment_summaries[:10])
        else:
            print("Segment Summaries:")
            print(segment_summaries)
        print(doc)
        print(process_id)
    segment_summaries = [segments.segmentlist(x) for x in segment_summaries]
    for x in segment_summaries:
        x.coalesce()
    clientutils.add_segment_info_ns(doc, process_id, segdefs, None, segment_summaries)

    # Do the same for excluded, except it seems that we skip everything related to the summaries, and don't end up
    #   adding the defs to the xml actually... they are just used to get the segments we will need for the result
    #   calculation...
    # For now, I'll just repeat what I've done before for the included, but not actually use the
    #   segmentdb_utils.add_segment_info command:

    if exclude_segments:
        ex_segdefs = []

        for excluded in exclude_segments.split(','):
            spec = excluded.split(':')

            if len(spec) < 2 or len(spec) > 3:
                print("Included segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr)
                sys.exit(1)

            ifo = spec[0]
            name = spec[1]
            # segdefs are actually segment summaries and need to look like this:
            # [('H1', 'DCH-TEST', 2, 1000000142, 1000000146, 0, 0), \
            #  ('H1', 'DCH-TEST', 1 , 999999999, 1000000020, 0, 0), \
            #  ('H1', 'DCH-TEST', 1, 1000000021, 1000000121, 0, 0), \
            #  ('H1', 'DCH-TEST', 1, 1000000136, 1000000139, 0, 0), \
            #  ('H1', 'DCH-TEST', 1, 1000000140, 1000000142, 0, 0)]
            # Of particular note: single version queries just use startTime and endTime, but versionless return the
            #   start and end time of the segment summaries that affect the final versionless result
            # This means that I need the versionless query above to return that additional information

            if len(spec) == 3 and spec[2] != '*':
                version = int(spec[2])
                if version < 1:
                    print("Segment version numbers must be greater than zero", file=sys.stderr)
                    sys.exit(1)
                # call to single version flag query goes here!
                # result is already processed into a python dict using json.loads :
                json_result, queryurl = apicalls.dqsegdbQueryTimesCompatible(protocol, server, ifo, name, version,
                                                                             include_list_string, startTime, endTime)
                excluded_JSON_results.append(json_result)
                ex_segdefs.append((ifo, name, version, startTime, endTime, 0, 0))
                #segment_summaries += json_result['known']&segments.segmentlist(segments.segment(startTime,endTime))
            else:
                version = '*'
                # Trigger versionless call to return a JSON result here
                json_result, versioned_JSON, affected_results = \
                    apicalls.dqsegdbCascadedQuery(protocol, server, ifo, name, include_list_string, startTime, endTime)
                # affected_results contains the segments of time that affected the final result for each version
                #   used in the calculation
                # This replicates the S6 behavior:
                excluded_JSON_results.append(json_result)
                # Don't actually need this next step for excluded, they don't end up in the summary table, and the
                #   ex_segdefs don't get written to the xml either
                for i_result in versioned_JSON:
                    excluded_intermediate_JSON.append(i_result)
                    for seg in affected_results[i_result['version']]:
                        ex_segdefs.append((ifo, name, i_result['version'], seg[0], seg[1], 0, 0))
                        # Ok, so here the segment_summaries are paired with the affected_results from above:  the
                        #   intersection of the affected results segments with each segment_summary for each version...
                        #segment_summaries +=
                        #    i_result['known']&segments.segmentlist(segments.segment(startTime,endTime))

    # Ok, now I have the included_JSON_results and the excluded_JSON_results, from which I can compute the equivalent
    #   of the old "found_segments" (included - excluded)
    if debug:
        import pdb
        pdb.set_trace()
    combined_result = clientutils.calculate_combined_result(included_JSON_results, excluded_JSON_results,
                                                            startTime, endTime, ifo)
    #found_segments = combined_result['active']
    found_segments = segments.segmentlist([segments.segment(x[0], x[1]) for x in combined_result['active']])
    if not options.strict_off:
        found_segments = found_segments & segments.segmentlist([segments.segment(startTime, endTime)])

    # Add the result type to the segment definer table
    seg_name = result_name
    seg_def_id = segmentdb_utils.add_to_segment_definer(doc, process_id, ifo, seg_name, 1)

    # and segment summary
    clientutils.add_to_segment_summary_ns(doc, process_id, seg_def_id, [[gps_start_time, gps_end_time]])

    # and store the segments
    clientutils.add_to_segment_ns(doc, process_id, seg_def_id, found_segments)
    if debug:
        print("Result name:")
        print(result_name)
        print("seg_def_id")
        print(seg_def_id)
        print("Found segments:")
        print(type(found_segments))
        print(found_segments)


#
# =============================================================================
#
#                                 XML/File routines
#
# =============================================================================
#

# ER6 Kipp API CHANGES:
class ContentHandler(ligolw.LIGOLWContentHandler):
    connection = None
#
# def setup_files(dir_name, gps_start_time, gps_end_time):
#     # Filter out the ones that are outside our time range
#@@ -477,7 +478,13 @@ def setup_files(dir_name, gps_start_time, gps_end_time):
#     target     = dbtables.get_connection_filename(temp_db, None, True, False)
#     connection = ligolw_sqlite.setup(target)
#
#-    ligolw_sqlite.insert_from_urls(connection, xml_files) # [temp_xml])
#+    ContentHandler.connection = connection
#+    dbtables.use_in(ContentHandler)
#+    # With ER6 Glue change:
#+    ligolw_sqlite.insert_from_urls(xml_files, ContentHandler)
#+
#+    # Pre-ER6 Kipp Glue change:
#+    #ligolw_sqlite.insert_from_urls(connection, xml_files) # [temp_xml])
####


def setup_files(dir_name, gps_start_time, gps_end_time):
    # Filter out the ones that are outside our time range
    xml_files = segmentdb_utils.get_all_files_in_range(dir_name, gps_start_time, gps_end_time)

    handle, temp_db = tempfile.mkstemp(suffix='.sqlite')
    os.close(handle)

    target = dbtables.get_connection_filename(temp_db, None, True, False)
    connection = ligolw_sqlite.setup(target)

    #ligolw_sqlite.insert_from_urls(connection, xml_files) # [temp_xml])
    ContentHandler.connection = connection
    dbtables.use_in(ContentHandler)
    ligolw_sqlite.insert_from_urls(xml_files, ContentHandler)

    segmentdb_utils.ensure_segment_table(connection)

    return temp_db, connection


#
# =============================================================================
#
#                                     Main
#
# =============================================================================
#

if __name__ == '__main__':
    options, database_location, file_location = parse_command_line()

    # Ping the database and exit if requested
    if options.ping:
        # DQSEGDB 'ping'
        o = urlparse(options.segment_url)
        protocol = o.scheme
        server = o.netloc
        ifo = "L1"
        url_result = urifunctions.constructFlagQueryURL(protocol, server, ifo)
        json_out = urifunctions.getDataUrllib2(url_result)
        flags_information = json.loads(json_out)
        server_version = flags_information['query_information']['api_version']
        print("Server version is %s" % server_version)
        sys.exit(0)

    #gps_start_time = int(options.gps_start_time)
    gps_start_time = float(options.gps_start_time)
    #gps_end_time = int(options.gps_end_time)
    gps_end_time = float(options.gps_end_time)

    # set up the response
    doc = ligolw.Document()
    doc.appendChild(ligolw.LIGO_LW())
    process_id = ligolw.Document.register_process(doc, PROGRAM_NAME, options.__dict__, version=__id__,
                                                  cvs_entry_time=__formatted_date__).process_id

    temp_files = False

    if database_location:
        # Use new dqsegdb client
        o = urlparse(options.segment_url)
        protocol = o.scheme
        server = o.netloc
        if options.show_types:
            run_show_types_dqsegdb(doc, protocol, server, gps_start_time, gps_end_time,
                                   options.include_segments, options.exclude_segments)

        if options.query_segments:
            run_query_segments_dqsegdb(doc, process_id, protocol, server, gps_start_time, gps_end_time,
                                       options.include_segments, options.exclude_segments, options.result_name)

        if options.query_types:
            warnings.warn("Notice: This call has not been optimized for DQSEGDB yet, and may take a few minutes.")
            run_query_types_dqsegdb(doc, process_id, protocol, server,
                                    gps_start_time, gps_end_time, options.include_segments)

    else:
        # Run old client
        warnings.warn('Warning: using old client code that may be less carefully maintained than DQSEGDB clients.')
        temp_db, connection = setup_files(file_location, gps_start_time, gps_end_time)
        engine = query_engine.SqliteQueryEngine(connection)
        temp_files = True
        if options.show_types:
            clientutils.run_show_types(doc, connection, engine, gps_start_time, gps_end_time,
                                       options.include_segments, options.exclude_segments)

        if options.query_segments:
            clientutils.run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time,
                                           options.include_segments, options.exclude_segments, options.result_name)

        if options.query_types:
            clientutils.run_query_types(doc, process_id, connection, engine, gps_start_time,
                                        gps_end_time, options.include_segments)

    # Write results
    gz = (options.output_file is not None and options.output_file.endswith(".gz"))
    utils.write_filename(doc, options.output_file, compress=gz)

    # Clean up
    if temp_files:
        os.remove(temp_db)
