#!/usr/bin/env pytest
###############################################################################
# $Id$
#
# Project:  GDAL/OGR Test Suite
# Purpose:  Test TileDB driver vector functionality.
# Author:   Even Rouault <even dot rouault at spatialys dot com>
#
###############################################################################
# Copyright (c) 2023, Even Rouault <even dot rouault at spatialys dot com>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
###############################################################################

import json
import math
import os
import shutil

import gdaltest
import pytest
import test_cli_utilities

from osgeo import gdal, ogr, osr

pytestmark = pytest.mark.require_driver("TileDB")

###############################################################################


def create_tiledb_dataset(nullable, batch_size, include_bool, extra_feature=False):

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    srs = osr.SpatialReference()
    srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
    srs.ImportFromEPSG(4326)
    options = []
    if batch_size:
        options += ["BATCH_SIZE=" + str(batch_size)]
    lyr = ds.CreateLayer("test", srs=srs, options=options)

    fld_defn = ogr.FieldDefn("strfield", ogr.OFTString)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("intfield", ogr.OFTInteger)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("int16field", ogr.OFTInteger)
    fld_defn.SetNullable(nullable)
    fld_defn.SetSubType(ogr.OFSTInt16)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("uint8field", ogr.OFTInteger)
    fld_defn.SetNullable(nullable)
    with gdal.config_option("TILEDB_INT_TYPE", "UINT8"):
        lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("uint16field", ogr.OFTInteger)
    fld_defn.SetNullable(nullable)
    with gdal.config_option("TILEDB_INT_TYPE", "UINT16"):
        lyr.CreateField(fld_defn)

    if include_bool:
        fld_defn = ogr.FieldDefn("boolfield", ogr.OFTInteger)
        fld_defn.SetNullable(nullable)
        fld_defn.SetSubType(ogr.OFSTBoolean)
        lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("int64field", ogr.OFTInteger64)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("doublefield", ogr.OFTReal)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("floatfield", ogr.OFTReal)
    fld_defn.SetNullable(nullable)
    fld_defn.SetSubType(ogr.OFSTFloat32)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("binaryfield", ogr.OFTBinary)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("intlistfield", ogr.OFTIntegerList)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("int16listfield", ogr.OFTIntegerList)
    fld_defn.SetNullable(nullable)
    fld_defn.SetSubType(ogr.OFSTInt16)
    lyr.CreateField(fld_defn)

    if include_bool:
        fld_defn = ogr.FieldDefn("boollistfield", ogr.OFTIntegerList)
        fld_defn.SetNullable(nullable)
        fld_defn.SetSubType(ogr.OFSTBoolean)
        lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("doublelistfield", ogr.OFTRealList)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("floatlistfield", ogr.OFTRealList)
    fld_defn.SetNullable(nullable)
    fld_defn.SetSubType(ogr.OFSTFloat32)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("datetimefield", ogr.OFTDateTime)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("datefield", ogr.OFTDate)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("timefield", ogr.OFTTime)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    fld_defn = ogr.FieldDefn("intfieldextra", ogr.OFTInteger)
    fld_defn.SetNullable(nullable)
    lyr.CreateField(fld_defn)

    field_count = lyr.GetLayerDefn().GetFieldCount()

    f = ogr.Feature(lyr.GetLayerDefn())
    f["strfield"] = "foo"
    f["intfield"] = -123456789
    f["int16field"] = -32768
    if include_bool:
        f["boolfield"] = True
    f["uint8field"] = 0
    f["uint16field"] = 0
    f["int64field"] = -1234567890123456
    f["doublefield"] = 1.2345
    f["floatfield"] = 1.5
    f.SetFieldBinaryFromHexString("binaryfield", "DEADBEEF")
    f["intlistfield"] = [-123456789, 123]
    f["int16listfield"] = [-32768, 32767]
    if include_bool:
        f["boollistfield"] = [True, False]
    f["doublelistfield"] = [1.2345, -1.2345]
    f["floatlistfield"] = [1.5, -1.5, 0]
    f["datetimefield"] = "2023-04-07T12:34:56.789Z"
    f["datefield"] = "2023-04-07"
    f["timefield"] = "12:34:56.789"
    f["intfieldextra"] = 1
    f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON ((1 2,1 3,4 3,1 2))"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 1

    f = ogr.Feature(lyr.GetLayerDefn())
    f["intfieldextra"] = 2
    f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON ((1 2,1 3,4 3,1 2))"))
    if not nullable:
        with gdaltest.error_handler():
            assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    else:
        assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 2

    f = ogr.Feature(lyr.GetLayerDefn())
    f["strfield"] = "barbaz"
    f["intfield"] = 123456789
    f["int16field"] = 32767
    if include_bool:
        f["boolfield"] = False
    f["uint8field"] = 255
    f["uint16field"] = 65535
    f["int64field"] = 1234567890123456
    f["doublefield"] = -1.2345
    f["floatfield"] = -1.5
    f.SetFieldBinaryFromHexString("binaryfield", "BEEFDEAD")
    f["intlistfield"] = [123456789, -123]
    f["int16listfield"] = [32767, -32768]
    if include_bool:
        f["boollistfield"] = [False, True]
    f["doublelistfield"] = [-1.2345, 1.2345]
    f["floatlistfield"] = [0.0, -1.5, 1.5]
    # Will be transformed to "2023/04/07 10:19:56.789+00"
    f["datetimefield"] = "2023-04-07T12:34:56.789+0215"
    f["datefield"] = "2023-04-08"
    f["timefield"] = "13:34:56.789"
    f["intfieldextra"] = 3
    f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON ((-10 -20,-1 -3,-4 -3,-10 -20))"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 3

    if extra_feature:
        f = ogr.Feature(lyr.GetLayerDefn())
        f["strfield"] = "something"
        f["intfield"] = 8765432
        f["int16field"] = 32767
        if include_bool:
            f["boolfield"] = False
        f["uint8field"] = 255
        f["uint16field"] = 65535
        f["int64field"] = 9876543210123456
        f["doublefield"] = -1.2345
        f["floatfield"] = -1.5
        f.SetFieldBinaryFromHexString("binaryfield", "DEAFBEEF")
        f["intlistfield"] = [-123456789, -123]
        f["int16listfield"] = [32767, -32768]
        if include_bool:
            f["boollistfield"] = [False, True]
        f["doublelistfield"] = [-1.2345, 1.2345]
        f["floatlistfield"] = [0.0, -1.5, 1.5]
        # Will be transformed to "2023/04/07 10:19:56.789+00"
        f["datetimefield"] = "2023-04-07T12:34:56.789+0215"
        f["datefield"] = "2023-04-08"
        f["timefield"] = "13:34:56.789"
        f["intfieldextra"] = 4
        f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (-0.9 -0.9)"))
        assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
        assert f.GetFID() == 4

    ds = None
    return field_count, srs, options


###############################################################################


@pytest.mark.parametrize("nullable,batch_size", [(True, None), (False, 2)])
def test_ogr_tiledb_basic(nullable, batch_size):

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    field_count, srs, options = create_tiledb_dataset(nullable, batch_size, True)

    ds = gdal.OpenEx("tmp/test.tiledb", open_options=options)
    lyr = ds.GetLayer(0)
    assert lyr.GetGeomType() == ogr.wkbUnknown
    assert lyr.GetSpatialRef().IsSame(srs)
    assert lyr.GetFeatureCount() == 3
    assert lyr.GetExtent() == (-10.0, 4.0, -20.0, 3.0)
    assert lyr.GetLayerDefn().GetFieldCount() == field_count
    for i in range(field_count):
        assert lyr.GetLayerDefn().GetFieldDefn(i).IsNullable() == nullable

    for i in range(3):
        f = lyr.GetNextFeature()
        if f.GetFID() == 1:
            assert f["strfield"] == "foo"
            assert f["intfield"] == -123456789
            assert f["int16field"] == -32768
            assert f["boolfield"] == True
            assert f["uint8field"] == 0
            assert f["uint16field"] == 0
            assert f["int64field"] == -1234567890123456
            assert f["doublefield"] == 1.2345
            assert f["floatfield"] == 1.5
            assert f.GetFieldAsBinary("binaryfield") == b"\xde\xad\xbe\xef"
            assert f["intlistfield"] == [-123456789, 123]
            assert f["int16listfield"] == [-32768, 32767]
            assert f["boollistfield"] == [True, False]
            assert f["doublelistfield"] == [1.2345, -1.2345]
            assert f["floatlistfield"] == [1.5, -1.5, 0]
            assert f["datetimefield"] == "2023/04/07 12:34:56.789+00"
            assert f["datefield"] == "2023/04/07"
            assert f["timefield"] == "12:34:56.789"
            assert f.GetGeometryRef().ExportToWkt() == "POLYGON ((1 2,1 3,4 3,1 2))"
        elif f.GetFID() == 2:
            assert f["intfieldextra"] == 2
            if nullable:
                for i in range(field_count):
                    if lyr.GetLayerDefn().GetFieldDefn(i).GetName() != "intfieldextra":
                        assert f.IsFieldNull(i)
            else:
                for i in range(field_count):
                    assert not f.IsFieldNull(i)
            assert f.GetGeometryRef().ExportToWkt() == "POLYGON ((1 2,1 3,4 3,1 2))"
        elif f.GetFID() == 3:
            assert f["strfield"] == "barbaz"
            assert f["intfield"] == 123456789
            assert f["int16field"] == 32767
            assert f["boolfield"] == False
            assert f["uint8field"] == 255
            assert f["uint16field"] == 65535
            assert f["int64field"] == 1234567890123456
            assert f["doublefield"] == -1.2345
            assert f["floatfield"] == -1.5
            assert f.GetFieldAsBinary("binaryfield") == b"\xbe\xef\xde\xad"
            assert f["intlistfield"] == [123456789, -123]
            assert f["int16listfield"] == [32767, -32768]
            assert f["boollistfield"] == [False, True]
            assert f["doublelistfield"] == [-1.2345, 1.2345]
            assert f["floatlistfield"] == [0.0, -1.5, 1.5]
            assert f["datetimefield"] == "2023/04/07 10:19:56.789+00"
            assert f["datefield"] == "2023/04/08"
            assert f["timefield"] == "13:34:56.789"
            assert (
                f.GetGeometryRef().ExportToWkt()
                == "POLYGON ((-10 -20,-1 -3,-4 -3,-10 -20))"
            )
        else:
            assert False

    f = lyr.GetNextFeature()
    assert f is None

    f = lyr.GetFeature(0)
    assert f is None

    f = lyr.GetFeature(3)
    assert f.GetFID() == 3
    assert f["strfield"] == "barbaz"

    lyr.SetSpatialFilterRect(0, 0, 10, 10)
    assert lyr.GetFeatureCount() == 2
    assert set(f.GetFID() for f in lyr) == set([1, 2])

    f = lyr.GetFeature(3)
    assert f.GetFID() == 3
    assert f["strfield"] == "barbaz"

    lyr.SetSpatialFilterRect(-10, -10, 0, 0)
    assert lyr.GetFeatureCount() == 1
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetSpatialFilterRect(100, 100, 110, 110)
    assert lyr.GetFeatureCount() == 0
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetSpatialFilter(None)

    lyr.SetAttributeFilter("strfield = 'foo'")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("'foo' = strfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("strfield = 'non_existing'")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("strfield <> 'foo'")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    res = set(f.GetFID() for f in lyr)
    assert 3 in res
    assert 1 not in res

    lyr.SetAttributeFilter("'foo' <> strfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    res = set(f.GetFID() for f in lyr)
    assert 3 in res
    assert 1 not in res

    lyr.SetAttributeFilter("intfield = -123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("intfield = -123456789.0")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("intfield = -9876543")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("intfield >= 123456790")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("123456790 <= intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("intfield >= 123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("123456789 <= intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("intfield > 123456788")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("123456788 < intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("intfield > 123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("123456789 < intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("intfield < -123456788")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("-123456788 > intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("-123456788 > intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("intfield < -123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("-123456789 > intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("intfield <= -123456790")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("-123456790 >= intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("intfield <= -123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("-123456789 >= intfield")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("boolfield = 1")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("boolfield = 0")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([3]) if nullable else set([2, 3]))

    # Out of domain
    lyr.SetAttributeFilter("boolfield = 2")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("boolfield <> 2")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    # Out of domain
    lyr.SetAttributeFilter("int16field = -32769")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("int16field <> -32769")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("int16field > -32769")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("int16field >= -32769")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("int16field < -32769")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("int16field <= -32769")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("int16field = 32768")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("int16field <> 32768")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("int16field < 32768")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("int16field <= 32768")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("int16field > 32768")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("int16field >= 32768")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("int64field = 1234567890123456")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("int64field = 1234567890123456.0")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("int64field > 2000000000")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([3])

    lyr.SetAttributeFilter("doublefield = 1.2345")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("doublefield = 1.2345999")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("doublefield = 1")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("floatfield = 1.5")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("datetimefield = '2023-04-07T12:34:56.789Z'")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    with pytest.raises(Exception):
        assert lyr.SetAttributeFilter("datetimefield = 'invalid'") == ogr.OGRERR_FAILURE

    lyr.SetAttributeFilter("datefield = '2023-04-07'")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    with pytest.raises(Exception):
        assert lyr.SetAttributeFilter("datefield = 'invalid'") == ogr.OGRERR_FAILURE

    lyr.SetAttributeFilter("timefield = '12:34:56.789'")
    # timefield comparison not supported by tiledb currently
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "NONE"
    assert set(f.GetFID() for f in lyr) == set([1])

    # Test AND
    lyr.SetAttributeFilter("int16field = -32768 AND intfield = -123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("int16field = 0 AND intfield = -123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("intfield = -123456789 AND int16field = 0")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set([])

    lyr.SetAttributeFilter("intfield = -123456789 AND (1 = 1)")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "PARTIAL"
    assert set(f.GetFID() for f in lyr) == set([1])

    lyr.SetAttributeFilter("(1 = 1) AND intfield = -123456789")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "PARTIAL"
    assert set(f.GetFID() for f in lyr) == set([1])

    # Test OR
    has_working_or_filter = (
        gdal.GetDriverByName("TileDB").GetMetadataItem("HAS_TILEDB_WORKING_OR_FILTER")
        != "NO"
    )
    if has_working_or_filter:
        lyr.SetAttributeFilter("intfield = 321 OR intfield = -123456789")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1])

        lyr.SetAttributeFilter("intfield = -123456789 OR intfield = 321")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1])

        lyr.SetAttributeFilter("intfield = 321 OR intfield = 123")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([])

        lyr.SetAttributeFilter("(1 = 1) OR intfield = -123456789")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "NONE"
        assert set(f.GetFID() for f in lyr) == set([1, 2, 3])

        lyr.SetAttributeFilter("(1 = 0) OR intfield = -123456789")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "NONE"
        assert set(f.GetFID() for f in lyr) == set([1])

        lyr.SetAttributeFilter("intfield = -123456789 OR (1 = 1)")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "NONE"
        assert set(f.GetFID() for f in lyr) == set([1, 2, 3])

        lyr.SetAttributeFilter("intfield = -123456789 OR (1 = 0)")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "NONE"
        assert set(f.GetFID() for f in lyr) == set([1])

    # Test NOT
    lyr.SetAttributeFilter("NOT (intfield = -123456789)")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([3]) if nullable else set([2, 3]))

    # Test IN
    if has_working_or_filter:
        lyr.SetAttributeFilter("intfield IN (321, -123456789)")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1])

        lyr.SetAttributeFilter("intfield IN (-123456789, 321)")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1])

    # Test IS NULL / IS NOT NULL
    lyr.SetAttributeFilter("strfield IS NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

    lyr.SetAttributeFilter("strfield IS NOT NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("intfield IS NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

    lyr.SetAttributeFilter("intfield IS NOT NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    # Test IS NULL and AND (for always_false situations)

    lyr.SetAttributeFilter("intfield IS NULL AND intfieldextra <> 4")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

    lyr.SetAttributeFilter("intfield IS NULL AND intfield IS NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

    lyr.SetAttributeFilter("intfieldextra <> 4 AND intfield IS NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

    lyr.SetAttributeFilter("intfield IS NULL AND intfieldextra = 4")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("intfieldextra = 4 AND intfield IS NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    # Test IS NOT NULL and AND (for always_true situations)

    lyr.SetAttributeFilter("intfield IS NOT NULL AND intfieldextra <> 4")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("intfield IS NOT NULL AND intfield IS NOT NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("intfieldextra <> 4 AND intfield IS NOT NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == (set([1, 3]) if nullable else set([1, 2, 3]))

    lyr.SetAttributeFilter("intfield IS NOT NULL AND intfieldextra = 4")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    lyr.SetAttributeFilter("intfieldextra = 4 AND intfield IS NOT NULL")
    assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
    assert set(f.GetFID() for f in lyr) == set()

    # Test IS NULL and OR (for always_false situations)
    if has_working_or_filter:
        lyr.SetAttributeFilter("intfield IS NULL OR intfieldextra <> 4")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1, 2, 3])

        lyr.SetAttributeFilter("intfield IS NULL OR intfield IS NULL")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

        lyr.SetAttributeFilter("intfieldextra <> 4 OR intfield IS NULL")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1, 2, 3])

        lyr.SetAttributeFilter("intfield IS NULL OR intfieldextra = 4")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

        lyr.SetAttributeFilter("intfieldextra = 4 OR intfield IS NULL")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == (set([2]) if nullable else set())

    # Test IS NOT NULL and OR (for always_true situations)
    if has_working_or_filter:
        lyr.SetAttributeFilter("intfield IS NOT NULL OR intfieldextra <> 4")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1, 2, 3])

        lyr.SetAttributeFilter("intfield IS NOT NULL OR intfield IS NOT NULL")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == (
            set([1, 3]) if nullable else set([1, 2, 3])
        )

        lyr.SetAttributeFilter("intfieldextra <> 4 OR intfield IS NOT NULL")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == set([1, 2, 3])

        lyr.SetAttributeFilter("intfield IS NOT NULL OR intfieldextra = 4")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == (
            set([1, 3]) if nullable else set([1, 2, 3])
        )

        lyr.SetAttributeFilter("intfieldextra = 4 OR intfield IS NOT NULL")
        assert lyr.GetMetadataItem("ATTRIBUTE_FILTER_TRANSLATION", "_DEBUG_") == "WHOLE"
        assert set(f.GetFID() for f in lyr) == (
            set([1, 3]) if nullable else set([1, 2, 3])
        )

    tiledb_md = json.loads(lyr.GetMetadata_List("json:TILEDB")[0])
    md = tiledb_md["array"]["metadata"]
    del md["CRS"]
    assert md == {
        "FEATURE_COUNT": {"type": "INT64", "value": 3},
        "FID_ATTRIBUTE_NAME": {"type": "STRING_UTF8", "value": "FID"},
        "GEOMETRY_ATTRIBUTE_NAME": {"type": "STRING_UTF8", "value": "wkb_geometry"},
        "GeometryType": {"type": "STRING_ASCII", "value": "Unknown"},
        "LAYER_EXTENT_MAXX": {"type": "FLOAT64", "value": 4.0},
        "LAYER_EXTENT_MAXY": {"type": "FLOAT64", "value": 3.0},
        "LAYER_EXTENT_MINX": {"type": "FLOAT64", "value": -10.0},
        "LAYER_EXTENT_MINY": {"type": "FLOAT64", "value": -20.0},
        "PAD_X": {"type": "FLOAT64", "value": 4.5},
        "PAD_Y": {"type": "FLOAT64", "value": 8.5},
    }

    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


@pytest.mark.parametrize(
    "wkt",
    [
        "POINT (1 2)",
        "POINT Z (1 2 3)",
        "POINT M (1 2 3)",
        "POINT ZM (1 2 3 4)",
        "LINESTRING (1 2,3 4)",
        "POLYGON ((0 0,0 1,1 1,0 0))",
        "MULTIPOINT ((0 0))",
        "MULTILINESTRING ((1 2,3 4))",
        "MULTIPOLYGON (((0 0,0 1,1 1,0 0)))",
        "GEOMETRYCOLLECTION (POINT (1 2))",
        "CIRCULARSTRING (0 0,1 1,2 0)",
        "COMPOUNDCURVE ((1 2,3 4))",
        "CURVEPOLYGON ((0 0,0 1,1 1,0 0))",
        "MULTICURVE ((1 2,3 4))",
        "MULTISURFACE (((0 0,0 1,1 1,0 0)))",
        "POLYHEDRALSURFACE (((0 0,0 1,1 1,0 0)))",
        "TIN (((0 0,0 1,1 1,0 0)))",
    ],
)
def test_ogr_tiledb_geometry_types(wkt):

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    g = ogr.CreateGeometryFromWkt(wkt)
    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    options = ["BOUNDS=-1e4,-1e4,1e4,1e4"]
    if g.GetGeometryType() in (ogr.wkbPoint, ogr.wkbPoint25D):
        options += ["GEOMETRY_NAME="]
    lyr = ds.CreateLayer("test", geom_type=g.GetGeometryType(), options=options)
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(g)
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    ds = None

    ds = ogr.Open("tmp/test.tiledb")
    lyr = ds.GetLayer(0)
    assert lyr.GetGeomType() == g.GetGeometryType()
    if g.GetGeometryType() in (ogr.wkbPoint, ogr.wkbPoint25D):
        assert lyr.GetGeometryColumn() == ""
    else:
        assert lyr.GetGeometryColumn() == "wkb_geometry"
    f = lyr.GetNextFeature()
    assert f.GetGeometryRef().ExportToIsoWkt() == wkt
    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


def test_ogr_tiledb_compression():

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    lyr = ds.CreateLayer(
        "test", options=["BOUNDS=-1e4,-1e4,1e4,1e4", "COMPRESSION=ZSTD"]
    )
    for (typ, subtype) in [
        (ogr.OFTInteger, ogr.OFSTNone),
        (ogr.OFTInteger, ogr.OFSTBoolean),
        (ogr.OFTInteger, ogr.OFSTInt16),
        (ogr.OFTReal, ogr.OFSTNone),
        (ogr.OFTReal, ogr.OFSTFloat32),
        (ogr.OFTInteger64, ogr.OFSTNone),
        (ogr.OFTIntegerList, ogr.OFSTNone),
        (ogr.OFTIntegerList, ogr.OFSTBoolean),
        (ogr.OFTIntegerList, ogr.OFSTInt16),
        (ogr.OFTRealList, ogr.OFSTNone),
        (ogr.OFTRealList, ogr.OFSTFloat32),
        (ogr.OFTInteger64List, ogr.OFSTNone),
        (ogr.OFTString, ogr.OFSTNone),
        (ogr.OFTBinary, ogr.OFSTNone),
        (ogr.OFTTime, ogr.OFSTNone),
        (ogr.OFTDate, ogr.OFSTNone),
        (ogr.OFTDateTime, ogr.OFSTNone),
    ]:
        fld_defn = ogr.FieldDefn("field%d_subtype%d" % (typ, subtype), typ)
        fld_defn.SetSubType(subtype)
        lyr.CreateField(fld_defn)
    ds = None

    ds = ogr.Open("tmp/test.tiledb")
    lyr = ds.GetLayer(0)
    tiledb_md = json.loads(lyr.GetMetadata_List("json:TILEDB")[0])
    ds = None

    assert tiledb_md["schema"]["coords_filter_list"] == ["ZSTD"]
    for attr in tiledb_md["schema"]["attributes"]:
        assert attr["filter_list"] == ["ZSTD"], attr

    shutil.rmtree("tmp/test.tiledb")


###############################################################################
# Run test_ogrsf


@pytest.mark.skipif(
    test_cli_utilities.get_test_ogrsf_path() is None, reason="test_ogrsf not available"
)
def test_ogr_tiledb_test_ogrsf():

    if os.path.exists("tmp/poly.tiledb"):
        shutil.rmtree("tmp/poly.tiledb")

    gdal.VectorTranslate("tmp/poly.tiledb", "data/poly.shp", format="TileDB")

    ret = gdaltest.runexternal(
        test_cli_utilities.get_test_ogrsf_path() + " tmp/poly.tiledb"
    )

    shutil.rmtree("tmp/poly.tiledb")

    assert "INFO" in ret
    assert "ERROR" not in ret


###############################################################################


def test_ogr_tiledb_dimension_names_open_option():

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    lyr = ds.CreateLayer(
        "test",
        geom_type=ogr.wkbPoint,
        options=["BOUNDS=-1e4,-1e4,1e4,1e4", "FID=", "GEOMETRY_NAME="],
    )
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 2)"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    ds = None

    ds = ogr.Open("tmp/test.tiledb")
    lyr = ds.GetLayer(0)
    f = lyr.GetNextFeature()
    assert f.GetGeometryRef().ExportToIsoWkt() == "POINT (1 2)"
    ds = None

    ds = gdal.OpenEx("tmp/test.tiledb", open_options=["DIM_X=_Y", "DIM_Y=_X"])
    lyr = ds.GetLayer(0)
    f = lyr.GetNextFeature()
    assert f.GetGeometryRef().ExportToIsoWkt() == "POINT (2 1)"
    ds = None

    with pytest.raises(Exception):
        gdal.OpenEx("tmp/test.tiledb", open_options=["DIM_X=invalid", "DIM_Y=_Y"])

    with pytest.raises(Exception):
        gdal.OpenEx(
            "tmp/test.tiledb",
            gdal.OF_UPDATE,
            open_options=["DIM_X=invalid", "DIM_Y=_Y"],
        )

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


def test_ogr_tiledb_switch_between_read_and_write():

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    lyr = ds.CreateLayer("test", options=["BOUNDS=-1e4,-1e4,1e4,1e4"])
    lyr.ResetReading()
    assert lyr.TestCapability(ogr.OLCSequentialWrite)
    assert lyr.TestCapability(ogr.OLCCreateField)
    assert lyr.CreateField(ogr.FieldDefn("intfield", ogr.OFTInteger)) == ogr.OGRERR_NONE
    f = ogr.Feature(lyr.GetLayerDefn())
    f["intfield"] = 1
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 2)"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 1

    assert lyr.TestCapability(ogr.OLCCreateField) == 0
    with pytest.raises(Exception):
        assert (
            lyr.CreateField(ogr.FieldDefn("intfield2", ogr.OFTInteger))
            == ogr.OGRERR_FAILURE
        )

    f = ogr.Feature(lyr.GetLayerDefn())
    f["intfield"] = 2
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (2 3)"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 2

    lyr.ResetReading()
    f = lyr.GetNextFeature()
    assert f.GetFID() == 1

    f = ogr.Feature(lyr.GetLayerDefn())
    f["intfield"] = 3
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (3 4)"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 3

    f = lyr.GetNextFeature()
    assert f.GetFID() == 1

    f = lyr.GetNextFeature()
    assert f.GetFID() == 2

    f = lyr.GetNextFeature()
    assert f.GetFID() == 3

    f = lyr.GetNextFeature()
    assert f is None

    ds = None

    ds = ogr.Open("tmp/test.tiledb", update=1)
    lyr = ds.GetLayer(0)
    assert lyr.TestCapability(ogr.OLCSequentialWrite)
    assert lyr.GetFeatureCount() == 3

    f = ogr.Feature(lyr.GetLayerDefn())
    f["intfield"] = 4
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (4 5)"))
    assert lyr.CreateFeature(f) == ogr.OGRERR_NONE
    assert f.GetFID() == 4

    f = lyr.GetNextFeature()
    assert f.GetFID() == 1

    f = lyr.GetNextFeature()
    assert f.GetFID() == 2

    f = lyr.GetNextFeature()
    assert f.GetFID() == 3

    f = lyr.GetNextFeature()
    assert f.GetFID() == 4

    f = lyr.GetNextFeature()
    assert f is None

    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


def test_ogr_tiledb_create_group():

    if "CREATE_GROUP" not in gdal.GetDriverByName("TileDB").GetMetadataItem(
        gdal.DMD_CREATIONOPTIONLIST
    ):
        pytest.skip("CREATE_GROUP not supported in TileDB < 2.9")

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource(
        "tmp/test.tiledb", options=["CREATE_GROUP=YES"]
    )
    assert ds.TestCapability(ogr.ODsCCreateLayer)
    lyr = ds.CreateLayer("test", options=["BOUNDS=-1e4,-1e4,1e4,1e4"])
    lyr.CreateField(ogr.FieldDefn("field", ogr.OFTString))
    assert ds.TestCapability(ogr.ODsCCreateLayer)
    lyr2 = ds.CreateLayer("test2", options=["BOUNDS=-1e4,-1e4,1e4,1e4"])
    lyr2.CreateField(ogr.FieldDefn("field2", ogr.OFTString))
    ds = None

    assert os.path.exists("tmp/test.tiledb/layers/test")
    assert os.path.exists("tmp/test.tiledb/layers/test2")

    ds = ogr.Open("tmp/test.tiledb")
    assert ds.GetLayerCount() == 2
    lyr = ds.GetLayerByName("test")
    assert lyr
    assert lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "field"
    lyr = ds.GetLayerByName("test2")
    assert lyr
    assert lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "field2"

    # Cannot create layer: read-only connection
    assert ds.TestCapability(ogr.ODsCCreateLayer) == 0
    with pytest.raises(Exception):
        ds.CreateLayer("failed", options=["BOUNDS=-1e4,-1e4,1e4,1e4"])
    ds = None

    ds = ogr.Open("tmp/test.tiledb", update=1)
    assert ds.TestCapability(ogr.ODsCCreateLayer)
    lyr = ds.CreateLayer("test/3", options=["BOUNDS=-1e4,-1e4,1e4,1e4"])
    assert lyr
    ds = None

    assert os.path.exists("tmp/test.tiledb/layers/test/3")

    ds = ogr.Open("tmp/test.tiledb")
    assert ds.GetLayerCount() == 3
    lyr = ds.GetLayerByName("test/3")
    assert lyr
    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


def test_ogr_tiledb_errors():

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")

    with pytest.raises(Exception):
        ds.CreateLayer("test", geom_type=ogr.wkbNone)

    with pytest.raises(Exception):
        ds.CreateLayer("test")  # missing bounds

    with pytest.raises(Exception):
        ds.CreateLayer("test", options=["BOUNDS=invalid"])

    lyr = ds.CreateLayer("test", options=["BOUNDS=1,2,3,4,5,6", "ADD_Z_DIM=YES"])

    with pytest.raises(Exception):
        ds.CreateLayer("another_layer", options=["BOUNDS=1,2,3,4,5,6"])

    lyr.CreateField(ogr.FieldDefn("foo", ogr.OFTString))
    for field_name in ("FID", "wkb_geometry", "_X", "_Y", "_Z", "foo"):
        # Existing field name
        with pytest.raises(Exception):
            lyr.CreateField(ogr.FieldDefn("FID", ogr.OFTString))

    with pytest.raises(Exception):
        # feature without geom
        lyr.CreateFeature(ogr.Feature(lyr.GetLayerDefn()))

    # feature with empty geom
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(ogr.Geometry(ogr.wkbPoint))
    with pytest.raises(Exception):
        lyr.CreateFeature(f)
    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


@pytest.mark.parametrize("nullable,batch_size", [(True, None), (False, 2)])
def test_ogr_tiledb_arrow_stream_pyarrow(nullable, batch_size):
    pytest.importorskip("pyarrow")

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    include_bool = "Boolean" in gdal.GetDriverByName("TileDB").GetMetadataItem(
        gdal.DMD_CREATIONFIELDDATASUBTYPES
    )
    _, _, options = create_tiledb_dataset(nullable, batch_size, include_bool)

    ds = gdal.OpenEx("tmp/test.tiledb", open_options=options)
    lyr = ds.GetLayer(0)

    mapFeatures = {}
    for f in lyr:
        mapFeatures[f.GetFID()] = f

    stream = lyr.GetArrowStreamAsPyArrow()
    schema = stream.schema
    fields = set(
        [
            (schema.field(i).name, str(schema.field(i).type))
            for i in range(schema.num_fields)
        ]
    )
    expected_fields = set(
        [
            ("FID", "int64"),
            ("strfield", "large_string"),
            ("intfield", "int32"),
            ("int16field", "int16"),
            ("int64field", "int64"),
            ("uint8field", "uint8"),
            ("uint16field", "uint16"),
            ("doublefield", "double"),
            ("floatfield", "float"),
            ("binaryfield", "large_binary"),
            ("intlistfield", "large_list<item: int32 not null>"),
            ("int16listfield", "large_list<item: int16 not null>"),
            ("doublelistfield", "large_list<item: double not null>"),
            ("floatlistfield", "large_list<item: float not null>"),
            ("datetimefield", "timestamp[ms]"),
            ("datefield", "date32[day]"),
            ("timefield", "time32[ms]"),
            ("intfieldextra", "int32"),
            ("wkb_geometry", "large_binary"),
        ]
    )
    if include_bool:
        expected_fields.add(("boolfield", "bool"))
        expected_fields.add(("boollistfield", "large_list<item: bool not null>"))
    assert fields == expected_fields

    def check_batch(batch):
        for idx, fid in enumerate(batch.field("FID")):
            f = mapFeatures[fid.as_py()]
            for field_idx in range(lyr.GetLayerDefn().GetFieldCount()):
                field_defn = lyr.GetLayerDefn().GetFieldDefn(field_idx)
                got_val = batch.field(field_defn.GetName())[idx].as_py()
                field_type = field_defn.GetType()
                if field_type == ogr.OFTDateTime:
                    if f.IsFieldSetAndNotNull(field_idx):
                        expected_val = f.GetFieldAsDateTime(field_idx)
                        assert [
                            got_val.year,
                            got_val.month,
                            got_val.day,
                            got_val.hour,
                            got_val.minute,
                            got_val.second + got_val.microsecond * 1e-6,
                        ] == pytest.approx(
                            expected_val[0:-1], abs=1e-4
                        ), field_defn.GetName()
                elif field_type == ogr.OFTBinary:
                    if f.IsFieldSetAndNotNull(field_idx):
                        got_val = bytes(got_val)
                        assert got_val == f.GetFieldAsBinary(
                            field_idx
                        ), field_defn.GetName()
                    else:
                        assert got_val is None, field_defn.GetName()
                elif field_type not in (
                    ogr.OFTDate,
                    ogr.OFTTime,
                ):
                    if isinstance(got_val, float) and math.isnan(got_val):
                        assert math.isnan(f.GetField(field_idx)), field_defn.GetName()
                    else:
                        assert got_val == f.GetField(field_idx), field_defn.GetName()

    for batch in stream:
        check_batch(batch)

    # Collect all batches (that is do not release them immediately)
    stream = lyr.GetArrowStreamAsPyArrow()
    batches = [batch for batch in stream]
    for batch in batches:
        check_batch(batch)

    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


@pytest.mark.parametrize("nullable,batch_size", [(True, None), (False, 2)])
def test_ogr_tiledb_arrow_stream_numpy(nullable, batch_size):
    pytest.importorskip("osgeo.gdal_array")
    numpy = pytest.importorskip("numpy")
    import datetime

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    include_bool = True
    _, _, options = create_tiledb_dataset(
        nullable, batch_size, include_bool, extra_feature=True
    )

    ds = gdal.OpenEx("tmp/test.tiledb", open_options=options)
    lyr = ds.GetLayer(0)

    mapFeatures = {}
    for f in lyr:
        mapFeatures[f.GetFID()] = f

    ds = gdal.OpenEx("tmp/test.tiledb", open_options=options)
    lyr = ds.GetLayer(0)

    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])

    def check_batch(batch):
        for idx, fid in enumerate(batch["FID"]):
            f = mapFeatures[fid]

            for field_idx in range(lyr.GetLayerDefn().GetFieldCount()):
                field_defn = lyr.GetLayerDefn().GetFieldDefn(field_idx)
                got_val = batch[field_defn.GetName()][idx]
                field_type = field_defn.GetType()
                if field_type in (ogr.OFTDateTime, ogr.OFTDate):
                    if f.IsFieldSetAndNotNull(field_idx):
                        expected_val = f.GetFieldAsDateTime(field_idx)
                        # Convert numpy.datetime64 to datetime.datetime
                        got_val = (
                            got_val - numpy.datetime64("1970-01-01T00:00:00")
                        ) / numpy.timedelta64(1, "s")
                        got_val = datetime.datetime.utcfromtimestamp(got_val)
                        assert [
                            got_val.year,
                            got_val.month,
                            got_val.day,
                            got_val.hour,
                            got_val.minute,
                            got_val.second + got_val.microsecond * 1e-6,
                        ] == pytest.approx(
                            expected_val[0:-1], abs=1e-4
                        ), field_defn.GetName()
                elif field_type == ogr.OFTString:
                    if f.IsFieldSetAndNotNull(field_idx):
                        assert got_val == f.GetField(field_idx).encode(
                            "UTF-8"
                        ), field_defn.GetName()
                    else:
                        assert len(got_val) == 0, field_defn.GetName()
                elif field_type == ogr.OFTBinary:
                    if f.IsFieldSetAndNotNull(field_idx):
                        got_val = bytes(got_val)
                        assert got_val == f.GetFieldAsBinary(
                            field_idx
                        ), field_defn.GetName()
                    else:
                        assert got_val is None, field_defn.GetName()
                else:
                    if f.IsFieldSetAndNotNull(field_idx):
                        if (
                            isinstance(got_val, numpy.float64)
                            or isinstance(got_val, numpy.float32)
                        ) and math.isnan(got_val):
                            assert math.isnan(
                                f.GetField(field_idx)
                            ), field_defn.GetName()
                        else:
                            expected_val = f.GetField(field_idx)
                            if isinstance(expected_val, list):
                                got_val = list(got_val)
                                assert got_val == expected_val, field_defn.GetName()

            got_geom = ogr.CreateGeometryFromWkb(batch["wkb_geometry"][idx])
            assert got_geom.ExportToIsoWkt() == f.GetGeometryRef().ExportToIsoWkt()

    for batch in stream:
        expected_fields = {
            "FID",
            "strfield",
            "intfield",
            "int16field",
            "int64field",
            "uint8field",
            "uint16field",
            "doublefield",
            "floatfield",
            "binaryfield",
            "intlistfield",
            "int16listfield",
            "doublelistfield",
            "floatlistfield",
            "datetimefield",
            "datefield",
            "timefield",
            "intfieldextra",
            "wkb_geometry",
        }
        if include_bool:
            expected_fields.add("boolfield")
            expected_fields.add("boollistfield")
        assert batch.keys() == expected_fields

        check_batch(batch)

    # Collect all batches (that is do not release them immediately)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    batches = [batch for batch in stream]
    assert len(batches) == (1 if batch_size is None else 2)

    for batch in batches:
        check_batch(batch)

    # Test spatial filter that intersects all features
    minx, maxx, miny, maxy = lyr.GetExtent()
    lyr.SetSpatialFilterRect(minx + 0.5, miny + 0.5, maxx - 0.5, maxy - 0.5)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    fids = []
    for batch in stream:
        fids += batch["FID"].tolist()
        check_batch(batch)
    assert set(fids) == set([1, 2, 3, 4])
    lyr.SetSpatialFilter(None)

    # Test spatial filter that intersects 1st, 2nd and 4th features only, but given how
    # spatial filtering works, more intermediate features will be collected
    # before being discarded
    lyr.SetSpatialFilterRect(-0.99, -2.99, 3, 3)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    fids = []
    for batch in stream:
        fids += batch["FID"].tolist()
        check_batch(batch)
    assert set(fids) == set([1, 2, 4])
    lyr.SetSpatialFilter(None)

    # Test spatial filter that intersects 1st and 2nd features only, but given how
    # spatial filtering works, more intermediate features will be collected
    # before being discarded
    lyr.SetSpatialFilterRect(0, 0, 3, 3)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    fids = []
    for batch in stream:
        fids += batch["FID"].tolist()
        check_batch(batch)
    assert set(fids) == set([1, 2])
    lyr.SetSpatialFilter(None)

    # Test spatial filter that intersects 3rd feature only
    lyr.SetSpatialFilterRect(-3, -3, -0.95, -0.95)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    fids = []
    for batch in stream:
        fids += batch["FID"].tolist()
        check_batch(batch)
    assert set(fids) == set([3])
    lyr.SetSpatialFilter(None)

    # Test spatial filter that intersects 4th feature only
    lyr.SetSpatialFilterRect(-0.95, -0.95, -0.85, -0.85)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    fids = []
    for batch in stream:
        fids += batch["FID"].tolist()
        check_batch(batch)
    assert set(fids) == set([4])
    lyr.SetSpatialFilter(None)

    # Test spatial filter that intersects no feature
    lyr.SetSpatialFilterRect(-0.5, -0.5, 0.5, 0.5)
    stream = lyr.GetArrowStreamAsNumPy(options=["USE_MASKED_ARRAYS=NO"])
    fids = []
    for batch in stream:
        fids += batch["FID"].tolist()
    assert set(fids) == set()
    lyr.SetSpatialFilter(None)

    stream = lyr.GetArrowStreamAsNumPy(
        options=["USE_MASKED_ARRAYS=NO", "INCLUDE_FID=NO", "MAX_FEATURES_IN_BATCH=1000"]
    )
    batches = [batch for batch in stream]
    assert len(batches) == 1
    batch = batches[0]
    assert "FID" not in batch.keys()
    strfield_values = [x.decode("UTF-8") for x in batch["strfield"]]
    assert "foo" in strfield_values
    assert "barbaz" in strfield_values
    got_geom = ogr.CreateGeometryFromWkb(batch["wkb_geometry"][0])
    assert got_geom

    lyr.SetIgnoredFields(["strfield"])
    stream = lyr.GetArrowStreamAsNumPy(
        options=["USE_MASKED_ARRAYS=NO", "MAX_FEATURES_IN_BATCH=1000"]
    )
    batches = [batch for batch in stream]
    assert len(batches) == 1
    batch = batches[0]
    assert "strfield" not in batch.keys()
    assert "intfield" in batch.keys()
    assert set([x for x in batch["intfield"]]) == set(
        [-123456789, 0, 123456789, 8765432]
    )

    lyr.SetIgnoredFields(["wkb_geometry"])
    stream = lyr.GetArrowStreamAsNumPy(
        options=["USE_MASKED_ARRAYS=NO", "MAX_FEATURES_IN_BATCH=1000"]
    )
    batches = [batch for batch in stream]
    assert len(batches) == 1
    batch = batches[0]
    assert "wkb_geometry" not in batch.keys()
    assert "intfield" in batch.keys()
    assert set([x for x in batch["intfield"]]) == set(
        [-123456789, 0, 123456789, 8765432]
    )

    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


def test_ogr_tiledb_arrow_stream_numpy_point_no_wkb_geometry_col():
    pytest.importorskip("osgeo.gdal_array")
    pytest.importorskip("numpy")

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    srs = osr.SpatialReference()
    srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
    srs.ImportFromEPSG(4326)
    lyr = ds.CreateLayer(
        "test", srs=srs, geom_type=ogr.wkbPoint, options=["GEOMETRY_NAME="]
    )
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 2)"))
    lyr.CreateFeature(f)
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (3 4)"))
    lyr.CreateFeature(f)
    ds = None

    ds = gdal.OpenEx("tmp/test.tiledb")
    lyr = ds.GetLayer(0)

    stream = lyr.GetArrowStreamAsNumPy()
    batches = [batch for batch in stream]
    assert len(batches) == 1
    batch = batches[0]
    for idx, fid in enumerate(batch["FID"]):
        got_geom = ogr.CreateGeometryFromWkb(batch["wkb_geometry"][idx])
        if fid == 1:
            assert got_geom.ExportToIsoWkt() == "POINT (1 2)"
        else:
            assert got_geom.ExportToIsoWkt() == "POINT (3 4)"

    ds = None

    shutil.rmtree("tmp/test.tiledb")


###############################################################################


def test_ogr_tiledb_arrow_stream_numpy_pointz_no_fid_and_wkb_geometry_col():
    pytest.importorskip("osgeo.gdal_array")
    pytest.importorskip("numpy")

    if os.path.exists("tmp/test.tiledb"):
        shutil.rmtree("tmp/test.tiledb")

    ds = ogr.GetDriverByName("TileDB").CreateDataSource("tmp/test.tiledb")
    srs = osr.SpatialReference()
    srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
    srs.ImportFromEPSG(4326)
    lyr = ds.CreateLayer(
        "test", srs=srs, geom_type=ogr.wkbPoint25D, options=["FID=", "GEOMETRY_NAME="]
    )
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT Z (1 2 3)"))
    lyr.CreateFeature(f)
    f = ogr.Feature(lyr.GetLayerDefn())
    f.SetGeometry(ogr.CreateGeometryFromWkt("POINT Z (4 5 6)"))
    lyr.CreateFeature(f)
    ds = None

    ds = gdal.OpenEx("tmp/test.tiledb")
    lyr = ds.GetLayer(0)

    stream = lyr.GetArrowStreamAsNumPy()
    batches = [batch for batch in stream]
    assert len(batches) == 1
    batch = batches[0]
    assert [x for x in batch["OGC_FID"]] == [1, 2]
    assert set(
        [ogr.CreateGeometryFromWkb(x).ExportToIsoWkt() for x in batch["wkb_geometry"]]
    ) == {"POINT Z (1 2 3)", "POINT Z (4 5 6)"}

    ds = None

    shutil.rmtree("tmp/test.tiledb")