trending/dbx
1
0
Fork 0
mirror of https://github.com/t8y2/dbx.git synced 2026-10-02 02:34:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(xugu): preserve spatial SRIDs in exports

Browse Source
This commit is contained in:
Elias
2026-08-25 15:15:41 +08:00
committed by GitHub
parent 343d006300
commit e9fea304ba
13 changed files with 669 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agents/drivers/xugu/main_test.go
+25
View File
@@ -1010,6 +1010,11 @@ func TestXuguIndexScopeDDL(t *testing.T) {
index: indexInfo{IndexType: &indexType},
want: " INDEXTYPE IS BTREE",
},
{
name: "spatial index preserves Xugu RTREE type",
index: indexInfo{IndexType: indexTypePtr("RTREE")},
want: " INDEXTYPE IS RTREE",
},
{
name: "local partition index",
index: indexInfo{IndexType: &indexType, IsLocal: true},
@@ -1082,6 +1087,26 @@ func TestXuguIndexScopeDDL(t *testing.T) {
}
}
func TestXuguIndexTypeName(t *testing.T) {
for _, tc := range []struct {
value any
want string
}{
{value: int64(0), want: "BTREE"},
{value: int64(1), want: "RTREE"},
{value: int64(2), want: "FULLTEXT"},
{value: int64(3), want: "BITMAP"},
{value: "RTREE", want: "RTREE"},
{value: "vendor-specific", want: "vendor-specific"},
} {
t.Run(fmt.Sprint(tc.value), func(t *testing.T) {
if got := indexTypeName(tc.value); got != tc.want {
t.Fatalf("indexTypeName(%v) = %q, want %q", tc.value, got, tc.want)
}
})
}
}
func indexTypePtr(value string) *string { return &value }
func TestXuguIndexPartitionDetailsStayInternalToTheGenericPayload(t *testing.T) {
agents/drivers/xugu/spatial_index_live_test.go
+89
View File
@@ -0,0 +1,89 @@
package main
import (
"os"
"strings"
"testing"
)
// TestLiveXuguSpatialIndexDDL verifies the Xugu-specific spatial index
// contract against a real server. Xugu reports spatial indexes as RTREE and
// reconstructed DDL must retain INDEXTYPE IS RTREE; emitting PostgreSQL's
// USING GIST syntax would not be executable on XuguDB.
func TestLiveXuguSpatialIndexDDL(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := connectParams{
Host: os.Getenv("XUGU_LIVE_HOST"),
Port: parsePort(os.Getenv("XUGU_LIVE_PORT")),
Database: os.Getenv("XUGU_LIVE_DATABASE"),
Username: os.Getenv("XUGU_LIVE_USERNAME"),
Password: os.Getenv("XUGU_LIVE_PASSWORD"),
}
if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" {
t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required")
}
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const (
table = "DBX_SPATIAL_INDEX_LIVE_T"
index = "DBX_SPATIAL_INDEX_LIVE_I"
)
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }()
for _, statement := range []string{
"CREATE TABLE " + table + " (ID INTEGER, GEOM GEOMETRY)",
"CREATE INDEX " + index + " ON " + table + " (GEOM) INDEXTYPE IS RTREE",
} {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
indexes, err := s.listIndexes(params.Username, table)
if err != nil {
t.Fatal(err)
}
var spatial *indexInfo
for i := range indexes {
if strings.EqualFold(indexes[i].Name, index) {
spatial = &indexes[i]
break
}
}
if spatial == nil {
t.Fatalf("spatial index %s was not listed: %#v", index, indexes)
}
if spatial.IndexType == nil || !strings.EqualFold(strings.TrimSpace(*spatial.IndexType), "RTREE") {
t.Fatalf("spatial index type = %#v, want RTREE: %#v", spatial.IndexType, *spatial)
}
if len(spatial.Columns) != 1 || !strings.EqualFold(spatial.Columns[0], "GEOM") {
t.Fatalf("spatial index columns = %#v, want [GEOM]", spatial.Columns)
}
ddl, err := s.getTableDDL(params.Username, table)
if err != nil {
t.Fatal(err)
}
upperDDL := strings.ToUpper(ddl)
if !strings.Contains(upperDDL, "CREATE INDEX") || !strings.Contains(upperDDL, strings.ToUpper(index)) {
t.Fatalf("table DDL omitted spatial index: %s", ddl)
}
if !strings.Contains(upperDDL, "INDEXTYPE IS RTREE") {
t.Fatalf("table DDL did not preserve Xugu RTREE syntax: %s", ddl)
}
if strings.Contains(upperDDL, "USING GIST") {
t.Fatalf("table DDL emitted PostgreSQL-only GIST syntax: %s", ddl)
}
}
agents/drivers/xugu/spatial_live_test.go
+186
View File
@@ -0,0 +1,186 @@
package main
import (
"fmt"
"os"
"strings"
"testing"
)
// TestLiveXuguSpatialQueryRegression exercises the value path used by the
// desktop map preview against a real Xugu server. It is opt-in because CI does
// not provide a Xugu service. The test deliberately covers both GEOMETRY and
// GEOGRAPHY columns, explicit SRIDs, EWKT input, and the multi-geometry types
// rendered by the shared map preview.
func TestLiveXuguSpatialQueryRegression(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := liveXuguParams(t)
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const table = "DBX_SPATIAL_VALUE_LIVE_T"
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }()
for _, statement := range []string{
"CREATE TABLE " + table + " (ID INTEGER, G_POINT GEOMETRY, G_MULTI GEOMETRY, G_MULTI_LINE GEOMETRY, G_COLLECTION GEOMETRY, G_GEO GEOGRAPHY)",
"INSERT INTO " + table + " VALUES (1, ST_GeomFromEWKT('SRID=4326;POINT(116.397 39.908)'), ST_GeomFromEWKT('SRID=3857;MULTIPOINT((1 2),(3 4))'), ST_GeomFromEWKT('SRID=4326;MULTILINESTRING((0 0,1 1),(2 2,3 3))'), ST_GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(3 4,5 6))'), ST_GeomFromEWKT('SRID=4326;POINT(116.397 39.908)'))",
"INSERT INTO " + table + " VALUES (2, ST_GeomFromText('LINESTRING(0 0,1 1,2 2)', 0), ST_GeomFromText('MULTIPOLYGON(((0 0,0 1,1 1,0 0)))', 4326), ST_GeomFromText('MULTILINESTRING((4 4,5 5),(6 6,7 7))', 4326), ST_GeomFromText('POLYGON((0 0,0 1,1 1,0 0))', 4326), ST_GeomFromText('POINT(10 20)', 4326))",
} {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
result, err := s.executeSelect("SELECT ID, G_POINT, G_MULTI, G_MULTI_LINE, G_COLLECTION, G_GEO FROM "+table+" ORDER BY ID", 20, 0)
if err != nil {
t.Fatal(err)
}
if len(result.Rows) != 2 {
t.Fatalf("row count = %d, want 2: %#v", len(result.Rows), result.Rows)
}
// The current Go driver reports direct GEOGRAPHY projections without a
// DatabaseTypeName. The decoder deliberately classifies a valid spatial
// payload as GEOMETRY in that case. This is sufficient for the shared map
// preview (which accepts GEOMETRY and GEOGRAPHY), while catalog metadata
// still exposes the source column as GEOGRAPHY.
wantTypes := []string{"INTEGER", "GEOMETRY", "GEOMETRY", "GEOMETRY", "GEOMETRY", "GEOMETRY"}
for i, want := range wantTypes {
if i >= len(result.ColumnTypes) || !strings.EqualFold(result.ColumnTypes[i], want) {
t.Fatalf("column %d type = %#v, want %s (columns=%#v)", i, result.ColumnTypes, want, result.Columns)
}
}
if len(result.SpatialColumns) != 5 {
t.Fatalf("spatial column metadata = %#v, want five geometry/geography columns", result.SpatialColumns)
}
for rowIndex, row := range result.Rows {
for _, columnIndex := range []int{1, 2, 3, 4, 5} {
if row[columnIndex] == nil {
t.Fatalf("row %d column %d unexpectedly NULL: %#v", rowIndex, columnIndex, row)
}
if !isXuguWKT(fmt.Sprint(row[columnIndex])) {
t.Fatalf("row %d column %d was not decoded as WKT: %#v", rowIndex, columnIndex, row[columnIndex])
}
}
}
if len(result.SpatialValues) != 2 || len(result.SpatialValues[0]) != 6 {
t.Fatalf("spatial cell metadata shape = %#v", result.SpatialValues)
}
if result.SpatialValues[0][1] == nil || *result.SpatialValues[0][1] != 4326 || result.SpatialValues[0][5] == nil || *result.SpatialValues[0][5] != 4326 {
t.Fatalf("row 1 SRIDs = %#v, want geometry/geography SRID 4326", result.SpatialValues[0])
}
if result.SpatialValues[0][2] == nil || *result.SpatialValues[0][2] != 3857 {
t.Fatalf("row 1 multipoint SRID = %#v, want 3857", result.SpatialValues[0][2])
}
raw, err := s.executeSelect("SELECT ST_AsBinary(G_POINT), ST_AsEWKB(G_POINT), ST_AsEWKT(G_GEO), ST_AsEWKB(G_MULTI), ST_AsEWKB(G_COLLECTION) FROM "+table+" WHERE ID = 1", 20, 0)
if err != nil {
t.Fatalf("WKB/EWKB/EWKT query failed: %v", err)
}
if len(raw.Rows) != 1 || len(raw.Rows[0]) != 5 {
t.Fatalf("unexpected WKB/EWKB/EWKT result: %#v", raw)
}
for columnIndex, wantSRID := range map[int]*uint32{0: nil, 1: uint32Pointer(4326)} {
bytes, ok := raw.Rows[0][columnIndex].(string)
if !ok {
t.Fatalf("raw WKB column %d type = %T, want binary string", columnIndex, raw.Rows[0][columnIndex])
}
decoded, decodedOK := decodeXuguWKB([]byte(bytes))
if !decodedOK || !isXuguWKT(decoded.WKT) || (wantSRID == nil && decoded.SRID != nil) || (wantSRID != nil && (decoded.SRID == nil || *decoded.SRID != *wantSRID)) {
t.Fatalf("raw WKB column %d decoded as value=%#v ok=%v", columnIndex, decoded, decodedOK)
}
}
decodedEWKT, sridEWKT, recognizedEWKT := decodeXuguSpatialValue(raw.Rows[0][2])
if !recognizedEWKT || decodedEWKT != "POINT(116.397 39.908)" || sridEWKT == nil || *sridEWKT != 4326 {
t.Fatalf("EWKT column decoded as value=%#v srid=%#v recognized=%v", decodedEWKT, sridEWKT, recognizedEWKT)
}
for columnIndex, want := range map[int]struct {
wkt string
srid uint32
}{
3: {wkt: "MULTIPOINT((1 2),(3 4))", srid: 3857},
4: {wkt: "GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(3 4,5 6))", srid: 4326},
} {
bytes, ok := raw.Rows[0][columnIndex].(string)
if !ok {
t.Fatalf("complex EWKB column %d type = %T, want binary string", columnIndex, raw.Rows[0][columnIndex])
}
decoded, decodedOK := decodeXuguWKB([]byte(bytes))
if !decodedOK || decoded.WKT != want.wkt || decoded.SRID == nil || *decoded.SRID != want.srid {
t.Fatalf("complex EWKB column %d decoded as value=%#v ok=%v", columnIndex, decoded, decodedOK)
}
}
t.Logf("Xugu WKB/EWKB/EWKT result: types=%#v columns=%#v rows=%#v spatialColumns=%#v spatialValues=%#v", raw.ColumnTypes, raw.Columns, raw.Rows, raw.SpatialColumns, raw.SpatialValues)
t.Logf("decoded Xugu spatial result: columns=%#v types=%#v spatialColumns=%#v values=%#v rows=%#v", result.Columns, result.ColumnTypes, result.SpatialColumns, result.SpatialValues, result.Rows)
}
// TestLiveXuguSpatialReplay probes the literal form generated by the generic
// export path. Xugu accepts WKT constructors for replay; the test records
// whether a plain quoted WKT is accepted so export code can remain database
// specific instead of guessing from PostgreSQL syntax.
func TestLiveXuguSpatialReplay(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := liveXuguParams(t)
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const table = "DBX_SPATIAL_REPLAY_LIVE_T"
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }()
for _, statement := range []string{
"CREATE TABLE " + table + " (ID INTEGER, G GEOMETRY, GEO GEOGRAPHY)",
"INSERT INTO " + table + " VALUES (1, ST_GeomFromEWKT('SRID=4326;POINT(1 2)'), ST_GeomFromEWKT('SRID=4326;POINT(3 4)'))",
} {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
if err := s.execWithReconnect("INSERT INTO " + table + " VALUES (2, 'POINT(5 6)', 'POINT(7 8)')"); err != nil {
t.Logf("plain quoted WKT is rejected by Xugu (expected on typed spatial columns): %v", err)
} else {
t.Log("plain quoted WKT is accepted by Xugu typed spatial columns")
}
if err := s.execWithReconnect("INSERT INTO " + table + " VALUES (3, ST_GeomFromEWKT('SRID=3857;POINT(9 10)'), ST_GeomFromEWKT('SRID=4326;POINT(11 12)'))"); err != nil {
t.Fatalf("Xugu spatial constructor replay failed: %v", err)
}
result, err := s.executeSelect("SELECT ID, ST_AsEWKT(G), ST_AsEWKT(GEO) FROM "+table+" ORDER BY ID", 20, 0)
if err != nil {
t.Fatal(err)
}
if len(result.Rows) < 2 {
t.Fatalf("spatial replay returned too few rows: %#v", result.Rows)
}
t.Logf("Xugu spatial replay result: types=%#v rows=%#v", result.ColumnTypes, result.Rows)
}
func liveXuguParams(t *testing.T) connectParams {
t.Helper()
params := connectParams{
Host: os.Getenv("XUGU_LIVE_HOST"),
Port: parsePort(os.Getenv("XUGU_LIVE_PORT")),
Database: os.Getenv("XUGU_LIVE_DATABASE"),
Username: os.Getenv("XUGU_LIVE_USERNAME"),
Password: os.Getenv("XUGU_LIVE_PASSWORD"),
}
if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" {
t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required")
}
return params
}
apps/desktop/src/components/grid/DataGrid.vue
+2
View File
@@ -7844,6 +7844,8 @@ const {
columnComments: visibleColumnComments,
allColumnComments,
mongoDocuments: computed(() => props.result.mongo_copy_documents ?? props.result.mongo_documents),
spatialColumns: computed(() => props.result.spatial_columns),
spatialValues: computed(() => props.result.spatial_values),
columnTypes: visibleColumnTypes,
allColumnTypes,
whereInput: computed(() => currentWhereInput()),
apps/desktop/src/composables/useDataGridExport.ts
+45 -5
View File
@@ -86,6 +86,8 @@ export interface UseDataGridExportOptions {
columnComments?: ComputedRef<Array<string | undefined>>;
allColumnComments?: ComputedRef<Array<string | undefined>>;
mongoDocuments?: ComputedRef<unknown[] | undefined>;
spatialColumns?: ComputedRef<QueryResult["spatial_columns"] | undefined>;
spatialValues?: ComputedRef<QueryResult["spatial_values"] | undefined>;
columnTypes: ComputedRef<Array<string | undefined> | undefined>;
allColumnTypes?: ComputedRef<Array<string | undefined> | undefined>;
whereInput: ComputedRef<string | undefined>;
@@ -174,6 +176,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
connectionId,
database,
context,
spatialColumns: spatialColumnsOption,
spatialValues: spatialValuesOption,
whereInput,
orderBy,
columnTypes,
@@ -335,12 +339,24 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
useFullExport = true,
formatDateTime = true,
headerMode: XlsxHeaderMode = "name",
): Promise<{ columns: string[]; columnTypes: string[]; columnComments?: (string | null)[]; rows: CellValue[][] }> {
): Promise<{
columns: string[];
columnTypes: string[];
columnComments?: (string | null)[];
spatialColumns?: QueryResult["spatial_columns"];
spatialValues?: QueryResult["spatial_values"];
rows: CellValue[][];
}> {
if (useFullExport && rowIds === undefined && fullExportResult && !hasCompleteLocalResult?.value) {
const result = await fullExportResult(onProgress);
if (result) {
const columnComments = buildXlsxHeaderOverrides(result.columns, commentsForExportColumns(result.columns), headerMode);
return { ...applyGlobalDateTimeExportFormat({ columns: result.columns, columnTypes: result.column_types ?? [], rows: result.rows }, formatDateTime), columnComments };
return {
...applyGlobalDateTimeExportFormat({ columns: result.columns, columnTypes: result.column_types ?? [], rows: result.rows }, formatDateTime),
columnComments,
spatialColumns: result.spatial_columns,
spatialValues: result.spatial_values,
};
}
}
// The full result is already in memory — export the raw QueryResult (all
@@ -351,19 +367,30 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
if (useFullExport && rowIds === undefined && hasCompleteLocalResult?.value && completeLocalResult?.value) {
const normalized = normalizeCompleteLocalResult(completeLocalResult.value);
const columnComments = buildXlsxHeaderOverrides(normalized.columns, normalized.columnComments, headerMode);
return { ...applyGlobalDateTimeExportFormat({ columns: normalized.columns, columnTypes: normalized.columnTypes, rows: normalized.rows }, formatDateTime), columnComments };
return {
...applyGlobalDateTimeExportFormat({ columns: normalized.columns, columnTypes: normalized.columnTypes, rows: normalized.rows }, formatDateTime),
columnComments,
spatialColumns: completeLocalResult.value.spatial_columns,
spatialValues: completeLocalResult.value.spatial_values,
};
}
const commentHeader = buildXlsxHeaderOverrides(columns.value, visibleXlsxColumnComments.value, headerMode);
const exportItems = await resolveVisibleRowValues(rowsToExport(rowIds));
const visibleIndexes = visibleColumnIndexesOption?.value ?? columns.value.map((_, index) => index);
const spatialColumns = spatialColumnsOption?.value?.map((column) => ({ column_index: visibleIndexes.indexOf(column.column_index), srid: column.srid })).filter((column) => column.column_index >= 0);
const spatialValues = spatialValuesOption?.value;
return {
...applyGlobalDateTimeExportFormat(
{
columns: columns.value,
columnTypes: (columnTypes.value ?? []).map((type) => type ?? ""),
rows: (await resolveVisibleRowValues(rowsToExport(rowIds))).map((item) => item.data),
rows: exportItems.map((item) => item.data),
},
formatDateTime,
),
columnComments: commentHeader,
...(spatialColumns?.length ? { spatialColumns } : {}),
...(spatialValues?.length ? { spatialValues: exportItems.map((item) => visibleIndexes.map((index) => spatialValues[item.sourceIndex ?? -1]?.[index] ?? null)) } : {}),
};
}
@@ -1278,6 +1305,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
tableName: tableMeta.value?.tableName || "table_name",
columns: exportData.columns,
columnTypes: exportData.columnTypes,
spatialColumns: exportData.spatialColumns,
spatialValues: exportData.spatialValues,
rows: exportData.rows,
});
await saveTextFile(content, exportFileName(tableMeta.value?.tableName || "export", "sql", { preferFallback: true }), "SQL", "sql");
@@ -1300,6 +1329,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
tableName: tableMeta.value?.tableName || "table_name",
columns: exportData.columns,
columnTypes: exportData.columnTypes,
spatialColumns: exportData.spatialColumns,
spatialValues: exportData.spatialValues,
rows: exportData.rows,
});
await saveTextFile(content, exportFileName("export-page", "sql", { page: true }), "SQL", "sql");
@@ -1315,17 +1346,26 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
await copyText(sql.value);
}
function sqlInsertExportData(result: { columns: string[]; rows: CellValue[][] }): {
function sqlInsertExportData(result: { columns: string[]; rows: CellValue[][]; spatialColumns?: QueryResult["spatial_columns"]; spatialValues?: QueryResult["spatial_values"] }): {
columns: string[];
columnTypes?: Array<string | undefined>;
spatialColumns?: QueryResult["spatial_columns"];
spatialValues?: QueryResult["spatial_values"];
rows: CellValue[][];
} {
const exportColumns = context.value === "table-data" && tableMeta.value ? effectiveColumns(sourceColumns.value, result.columns) : result.columns;
const columnIndexes = exportColumns.map((column, index) => ({ column, index })).filter((item): item is { column: string; index: number } => !!item.column);
const exportColumnTypes = columnTypes.value?.length === result.columns.length ? columnTypes.value : undefined;
const indexBySource = new Map(columnIndexes.map((item, index) => [item.index, index]));
const spatialColumns = result.spatialColumns?.flatMap((column) => {
const columnIndex = indexBySource.get(column.column_index);
return columnIndex === undefined ? [] : [{ column_index: columnIndex, srid: column.srid }];
});
return {
columns: columnIndexes.map((item) => item.column),
columnTypes: exportColumnTypes ? columnIndexes.map((item) => exportColumnTypes[item.index]) : undefined,
...(spatialColumns?.length ? { spatialColumns } : {}),
...(result.spatialValues?.length ? { spatialValues: result.spatialValues.map((row) => columnIndexes.map((item) => row[item.index] ?? null)) } : {}),
rows: result.rows.map((row) => columnIndexes.map((item) => row[item.index] ?? null)),
};
}
apps/desktop/src/lib/__tests__/dataGrid/geometryMapPreview.spec.ts
Vendored
+23
View File
@@ -29,6 +29,29 @@ function context(): PreviewActionContext {
}
describe("buildGeometryMapFeatureCollection", () => {
it("treats GEOGRAPHY columns as map-preview layers and preserves SRID metadata", () => {
const ctx = context();
ctx.result.column_types = ["GEOGRAPHY", "text"];
ctx.result.columns = ["location", "name"];
ctx.result.rows = [["SRID=4326;POINT(116.397 39.908)", "wgs84"]];
ctx.result.spatial_columns = [{ column_index: 0, srid: 4326 }];
ctx.result.spatial_values = [[4326, null]];
ctx.displayRowRefs = [{ id: 1, sourceIndex: 0, isNew: false }];
expect(hasGeometryMapPreviewColumns(ctx.result)).toBe(true);
expect(buildGeometryMapFeatureCollection(ctx)).toEqual({
type: "FeatureCollection",
detectedSrid: 4326,
features: [
{
type: "Feature",
geometry: { type: "Point", coordinates: [116.397, 39.908] },
properties: { _column: "location", _row: 0, _srid: 4326, name: "wgs84" },
},
],
});
});
it("checks preview availability from column metadata without parsing result rows", () => {
const result = context().result;
expect(hasGeometryMapPreviewColumns(result)).toBe(true);
apps/desktop/src/lib/export/databaseExport.ts
+4
View File
@@ -18,6 +18,8 @@ export interface ExportedTableSql {
columns: string[];
columnTypes?: Array<string | null | undefined>;
columnExtras?: Array<string | null | undefined>;
spatialColumns?: QueryResult["spatial_columns"];
spatialValues?: QueryResult["spatial_values"];
rows: QueryResult["rows"];
truncated?: boolean;
}
@@ -41,6 +43,8 @@ export interface BuildExportInsertStatementsOptions {
columns: string[];
columnTypes?: Array<string | null | undefined>;
columnExtras?: Array<string | null | undefined>;
spatialColumns?: QueryResult["spatial_columns"];
spatialValues?: QueryResult["spatial_values"];
rows: QueryResult["rows"];
batchSize?: number;
}
apps/desktop/src/lib/export/exportFormats.ts
+3 -1
View File
@@ -1,4 +1,4 @@
import type { DatabaseType } from "@/types/database";
import type { DatabaseType, QueryResult } from "@/types/database";
import * as api from "@/lib/backend/api";
export type ExportCellValue = string | number | boolean | null;
@@ -35,6 +35,8 @@ export interface FormatSqlInsertOptions {
qualifiedTableName?: string;
columns: string[];
columnTypes?: Array<string | null | undefined>;
spatialColumns?: QueryResult["spatial_columns"];
spatialValues?: QueryResult["spatial_values"];
rows: ExportCellValue[][];
}
crates/dbx-core/examples/data_transfer_bench.rs
+2
View File
@@ -100,6 +100,8 @@ fn run() -> Result<(), String> {
columns: columns.clone(),
column_types: vec![None; columns.len()],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: rows.clone(),
batch_size: Some(options.batch_size),
})?;
crates/dbx-core/src/database_export.rs
+260 -14
View File
@@ -17,7 +17,7 @@ use crate::transfer::{
is_mysql_generated_column_extra, keyset_pagination_sql_with_identifier_quote, quote_identifier,
quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table,
};
use crate::types::ObjectSourceKind;
use crate::types::{ObjectSourceKind, SpatialColumn};
static EXPORT_CANCELLED: std::sync::LazyLock<RwLock<HashSet<String>>> =
std::sync::LazyLock::new(|| RwLock::new(HashSet::new()));
@@ -288,6 +288,10 @@ pub struct ExportedTableSql {
#[serde(default)]
pub column_extras: Vec<Option<String>>,
#[serde(default)]
pub spatial_columns: Vec<SpatialColumn>,
#[serde(default)]
pub spatial_values: Vec<Vec<Option<u32>>>,
#[serde(default)]
pub rows: Vec<Vec<Value>>,
#[serde(default)]
pub truncated: bool,
@@ -313,6 +317,10 @@ pub struct BuildExportInsertStatementsOptions {
#[serde(default)]
pub column_extras: Vec<Option<String>>,
#[serde(default)]
pub spatial_columns: Vec<SpatialColumn>,
#[serde(default)]
pub spatial_values: Vec<Vec<Option<u32>>>,
#[serde(default)]
pub rows: Vec<Vec<Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub batch_size: Option<usize>,
@@ -394,6 +402,9 @@ fn format_export_sql_literal_typed(
if let Some(literal) = format_mysql_spatial_export_literal(value, database_type, column_type) {
return literal;
}
if let Some(literal) = format_xugu_spatial_export_literal(value, database_type, column_type) {
return literal;
}
if is_mysql_compatible_export_literal_target(database_type) {
if column_type.is_some_and(is_mysql_binary_export_type) {
if let Some(literal) = format_mysql_binary_export_literal(value) {
@@ -832,6 +843,50 @@ pub(crate) fn is_mysql_spatial_export_type(column_type: &str) -> bool {
)
}
pub(crate) fn is_xugu_spatial_export_type(column_type: &str) -> bool {
let normalized = column_type.trim().to_ascii_lowercase();
let base = normalized.split(['(', ':', ' ', '\t', '\n']).next().unwrap_or("").trim();
matches!(base, "geometry" | "geography")
}
/// Xugu returns spatial values as readable WKT/EWKT text. Plain WKT is
/// accepted by the server, but it cannot carry a non-zero SRID; database
/// exports therefore select EWKT and replay it through the Xugu constructor.
/// This branch is intentionally Xugu-only so PostgreSQL/PostGIS and other
/// dialects retain their existing export behavior.
pub(crate) fn format_xugu_spatial_export_literal(
value: &Value,
database_type: Option<DatabaseType>,
column_type: Option<&str>,
) -> Option<String> {
format_xugu_spatial_export_literal_with_srid(value, database_type, column_type, None)
}
fn format_xugu_spatial_export_literal_with_srid(
value: &Value,
database_type: Option<DatabaseType>,
column_type: Option<&str>,
srid: Option<u32>,
) -> Option<String> {
if database_type != Some(DatabaseType::Xugu) || !column_type.is_some_and(is_xugu_spatial_export_type) {
return None;
}
if value.is_null() {
return Some("NULL".to_string());
}
let text = value.as_str().map_or_else(|| value.to_string(), ToString::to_string);
let trimmed = text.trim_start();
if trimmed.len() > 5 && trimmed[..5].eq_ignore_ascii_case("SRID=") {
return Some(format!("ST_GeomFromEWKT({})", quote_export_sql_string(&text)));
}
if let Some(srid) = srid.filter(|srid| *srid != 0) {
return Some(format!("ST_GeomFromEWKT({})", quote_export_sql_string(&format!("SRID={srid};{text}"))));
}
// Xugu accepts a plain WKT string for both GEOMETRY and GEOGRAPHY. Keep
// that form for SRID 0/legacy values rather than inventing a constructor.
Some(quote_export_sql_string(&text))
}
/// Database exports encode MySQL spatial cells as `DBX_WKB:<srid>:<hex>` while
/// reading them. Keeping this marker internal lets the normal JSON row shape
/// and all non-export query paths continue to expose readable WKT values.
@@ -926,6 +981,31 @@ fn format_export_numeric_literal(value: &Value) -> Option<String> {
}
}
fn export_column_type<'a>(
column_types: &'a [Option<String>],
index: usize,
database_type: Option<DatabaseType>,
spatial_columns: &HashMap<usize, Option<u32>>,
) -> Option<&'a str> {
column_types.get(index).and_then(|value| value.as_deref()).filter(|value| !value.trim().is_empty()).or_else(|| {
(database_type == Some(DatabaseType::Xugu) && spatial_columns.contains_key(&index)).then_some("GEOMETRY")
})
}
fn format_export_sql_literal_typed_with_spatial(
value: &Value,
database_type: Option<DatabaseType>,
column_type: Option<&str>,
sqlserver_unicode_string: bool,
spatial_srid: Option<u32>,
) -> String {
if let Some(literal) = format_xugu_spatial_export_literal_with_srid(value, database_type, column_type, spatial_srid)
{
return literal;
}
format_export_sql_literal_typed(value, database_type, column_type, sqlserver_unicode_string)
}
fn is_export_numeric_literal(text: &str) -> bool {
if text.trim() != text || text.is_empty() {
return false;
@@ -947,12 +1027,14 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
options.qualified_table_name.as_deref(),
options.identifier_quote.as_deref(),
)?;
let spatial_columns =
options.spatial_columns.iter().map(|column| (column.column_index, column.srid)).collect::<HashMap<_, _>>();
let insert_columns = options
.columns
.iter()
.enumerate()
.filter_map(|(index, column)| {
let column_type = options.column_types.get(index).and_then(|value| value.as_deref());
let column_type = export_column_type(&options.column_types, index, options.database_type, &spatial_columns);
is_export_insert_column(
options.database_type,
column,
@@ -1023,7 +1105,7 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
*row_count = 0;
};
for row in options.rows {
for (row_index, row) in options.rows.into_iter().enumerate() {
let mut rendered_row = String::with_capacity(insert_columns.len().saturating_mul(16).saturating_add(2));
rendered_row.push('(');
for (column_index, (index, _, sqlserver_unicode_string)) in insert_columns.iter().enumerate() {
@@ -1031,11 +1113,21 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
rendered_row.push_str(", ");
}
let value = row.get(*index).unwrap_or(&Value::Null);
rendered_row.push_str(&format_export_sql_literal_typed(
let column_type =
export_column_type(&options.column_types, *index, options.database_type, &spatial_columns);
let spatial_srid = options
.spatial_values
.get(row_index)
.and_then(|values| values.get(*index))
.copied()
.flatten()
.or_else(|| spatial_columns.get(index).copied().flatten());
rendered_row.push_str(&format_export_sql_literal_typed_with_spatial(
value,
options.database_type,
options.column_types.get(*index).and_then(|value| value.as_deref()),
column_type,
*sqlserver_unicode_string,
spatial_srid,
));
}
rendered_row.push(')');
@@ -1164,6 +1256,8 @@ pub fn build_database_sql_export(options: BuildDatabaseSqlExportOptions) -> Resu
columns: table.columns,
column_types: table.column_types,
column_extras: table.column_extras,
spatial_columns: table.spatial_columns,
spatial_values: table.spatial_values,
rows: table.rows,
batch_size: Some(insert_batch_size),
})?;
@@ -1627,6 +1721,11 @@ fn database_export_select_list(columns: &[String], column_types: &[Option<String
&& column_types.get(index).and_then(|value| value.as_deref()).is_some_and(is_mysql_spatial_export_type)
{
mysql_spatial_export_marker_expression(column)
} else if *db_type == DatabaseType::Xugu
&& column_types.get(index).and_then(|value| value.as_deref()).is_some_and(is_xugu_spatial_export_type)
{
let quoted = quote_identifier(column, db_type);
format!("ST_AsEWKT({quoted}) AS {quoted}")
} else {
quote_identifier(column, db_type)
}
@@ -1649,7 +1748,7 @@ pub(crate) fn replace_database_export_select_list(
let prefix = format!("SELECT {original}");
if !sql.starts_with(&prefix) {
log::warn!(
"MySQL spatial database export could not replace its SELECT list; geometry columns will be exported as WKT"
"Spatial database export could not replace its SELECT list; geometry columns may lose SRID metadata"
);
return sql;
}
@@ -1735,6 +1834,8 @@ fn write_database_export_rows<W: Write>(
columns: insert_columns.to_vec(),
column_types: insert_column_types.to_vec(),
column_extras: insert_column_extras.to_vec(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: insert_rows.to_vec(),
batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE),
})?;
@@ -2967,18 +3068,20 @@ mod tests {
database_export_select_sql, database_export_total_objects, drop_table_if_exists_sql,
ensure_export_destination_dir, export_destination_identity_mismatch, filter_export_table_infos,
format_export_sql_literal, format_export_table_ddl, format_mysql_spatial_export_literal,
generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl,
generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble,
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl,
record_export_destination_identity, record_export_error, replace_database_export_select_list,
sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows,
BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, DatabaseExportObjectCounts,
DatabaseExportRequest, DdlNormalizeOptions, ExportedTableSql, PostgresExportExtension, PostgresExportSequence,
PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
format_xugu_spatial_export_literal, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl,
generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql,
is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows,
mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity,
record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies,
split_postgres_export_table_triggers, write_database_export_rows, BuildDatabaseSqlExportOptions,
BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions,
ExportedTableSql, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers,
DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
};
use crate::connection::AppState;
use crate::models::connection::DatabaseType;
use crate::storage::Storage;
use crate::types::SpatialColumn;
use crate::types::{ObjectInfo, ObjectSourceKind, TableInfo};
use serde_json::{json, Value};
use std::sync::{
@@ -3441,6 +3544,8 @@ mod tests {
columns: vec!["location".to_string(), "shape".to_string()],
column_types: vec![Some("point".to_string()), Some("geometry".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![
json!("DBX_WKB:4326:0101000000AE47E17A14AE5C4052B81E85EBF34240"),
json!("DBX_WKB:0:0101000000000000000000F03F0000000000000040"),
@@ -3508,6 +3613,85 @@ mod tests {
assert!(sql.contains("ORDER BY `id` ASC LIMIT 1000"), "sql: {sql}");
}
#[test]
fn xugu_spatial_export_selects_ewkt_to_preserve_srid() {
let sql = database_export_select_sql(
&["id".to_string(), "shape".to_string(), "location".to_string(), "name".to_string()],
&[
Some("INTEGER".to_string()),
Some("GEOMETRY".to_string()),
Some("GEOGRAPHY".to_string()),
Some("VARCHAR(32)".to_string()),
],
"places",
"app",
&DatabaseType::Xugu,
);
assert_eq!(
sql,
"SELECT \"id\", ST_AsEWKT(\"shape\") AS \"shape\", ST_AsEWKT(\"location\") AS \"location\", \"name\" FROM \"app\".\"places\""
);
}
#[test]
fn xugu_spatial_export_replays_ewkt_and_keeps_plain_wkt_compatible() {
assert_eq!(
format_xugu_spatial_export_literal(
&json!("SRID=3857;POINT(1 2)"),
Some(DatabaseType::Xugu),
Some("GEOMETRY"),
),
Some("ST_GeomFromEWKT('SRID=3857;POINT(1 2)')".to_string())
);
assert_eq!(
format_xugu_spatial_export_literal(&json!("POINT(1 2)"), Some(DatabaseType::Xugu), Some("GEOGRAPHY"),),
Some("'POINT(1 2)'".to_string())
);
assert_eq!(
format_xugu_spatial_export_literal(&Value::Null, Some(DatabaseType::Xugu), Some("GEOMETRY")),
Some("NULL".to_string())
);
assert!(format_xugu_spatial_export_literal(
&json!("SRID=3857;POINT(1 2)"),
Some(DatabaseType::Xugu),
Some("VARCHAR")
)
.is_none());
assert!(format_xugu_spatial_export_literal(
&json!("SRID=3857;POINT(1 2)"),
Some(DatabaseType::Postgres),
Some("GEOMETRY")
)
.is_none());
}
#[test]
fn xugu_spatial_metadata_recovers_missing_type_and_cell_srid() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Xugu),
identifier_quote: None,
schema: Some("app".to_string()),
table_name: Some("places".to_string()),
qualified_table_name: None,
columns: vec!["shape".to_string()],
column_types: vec![None],
column_extras: vec![None],
spatial_columns: vec![SpatialColumn { column_index: 0, srid: Some(3857) }],
spatial_values: vec![vec![Some(3857)]],
rows: vec![vec![json!("POINT(1 2)")]],
batch_size: None,
})
.unwrap();
assert_eq!(
statements,
vec![
r#"INSERT INTO "app"."places" ("shape") VALUES (ST_GeomFromEWKT('SRID=3857;POINT(1 2)'));"#.to_string()
]
);
}
#[test]
fn database_specific_boolean_export_literals() {
let sqlserver_statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
@@ -3519,6 +3703,8 @@ mod tests {
columns: vec!["typed_true".to_string(), "untyped_false".to_string(), "typed_null".to_string()],
column_types: vec![Some("bit".to_string()), None, Some("bit".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(true), json!(false), Value::Null]],
batch_size: Some(10),
})
@@ -3532,6 +3718,8 @@ mod tests {
columns: vec!["enabled".to_string(), "disabled".to_string(), "unknown".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(true), json!(false), Value::Null]],
batch_size: Some(10),
})
@@ -3574,6 +3762,8 @@ mod tests {
Some("nvarchar(20)".to_string()),
],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![
json!("张'三"),
json!("中文"),
@@ -3605,6 +3795,8 @@ mod tests {
columns: vec!["body".to_string()],
column_types: vec![Some("text".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("line1\nline2\tcol\rend\\slash\0\x1aO'Hara")]],
batch_size: Some(10),
})
@@ -3628,6 +3820,8 @@ mod tests {
columns: vec!["payload".to_string()],
column_types: vec![Some("longtext".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(long_value.clone())], vec![json!(long_value)]],
batch_size: Some(100),
})
@@ -3648,6 +3842,8 @@ mod tests {
columns: vec!["id".to_string()],
column_types: vec![Some("int".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: (0..1001).map(|id| vec![json!(id)]).collect(),
batch_size: Some(2000),
})
@@ -3669,6 +3865,8 @@ mod tests {
columns: vec!["message".to_string()],
column_types: vec![Some("varchar(255)".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("first\nsecond\tthird")]],
batch_size: Some(10),
})
@@ -3688,6 +3886,8 @@ mod tests {
columns: vec!["body".to_string()],
column_types: vec![Some("text".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("line1\nline2\tend")]],
batch_size: Some(10),
})
@@ -3712,6 +3912,8 @@ mod tests {
Some("text".to_string()),
],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("line1\rline2"), json!("O'Hara"), json!(r"C:\tmp"), json!("plain")]],
batch_size: Some(10),
})
@@ -3736,6 +3938,8 @@ mod tests {
columns: vec!["payload".to_string()],
column_types: vec![Some("jsonb".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(r#"{"text":"say \"hi\"","path":"C:\\tmp","quote":"O'Hara"}"#)]],
batch_size: Some(10),
})
@@ -3770,6 +3974,8 @@ mod tests {
Some("text[]".to_string()),
],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!([1.2, 3.4]), json!(["5", "6"]), json!(["x", "y"])]],
batch_size: Some(10),
})
@@ -3794,6 +4000,8 @@ mod tests {
columns: vec!["id".to_string(), "name".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Hara")], vec![json!(3), json!("Linus")]],
batch_size: Some(2),
})
@@ -3819,6 +4027,8 @@ mod tests {
columns: vec!["ID".to_string(), "NAME".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]],
batch_size: Some(100),
})
@@ -3844,6 +4054,8 @@ mod tests {
columns: vec!["__DBX_ROWID".to_string(), "ID".to_string(), "NAME".to_string()],
column_types: vec![Some("VARCHAR2".to_string()), Some("NUMBER".to_string()), Some("VARCHAR2".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("AAAPr9AAEAAAAGfAAA"), json!(1), json!("Ada")]],
batch_size: Some(100),
})
@@ -3863,6 +4075,8 @@ mod tests {
columns: vec!["__DBX_ROWID".to_string(), "ID".to_string(), "NAME".to_string()],
column_types: vec![Some("VARCHAR2".to_string()), Some("NUMBER".to_string()), Some("VARCHAR2".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("*AAABk1AAEAAAAAgAAA"), json!(1), json!("Ada")]],
batch_size: Some(100),
})
@@ -3882,6 +4096,8 @@ mod tests {
columns: vec!["__DBX_ROWID".to_string(), "name".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(7), json!("Ada")]],
batch_size: Some(100),
})
@@ -3901,6 +4117,8 @@ mod tests {
columns: vec!["ID".to_string(), "CREATED_ON".to_string(), "RAW_TEXT".to_string()],
column_types: vec![Some("NUMBER".to_string()), Some("DATE".to_string()), Some("VARCHAR2(64)".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")],
vec![json!(2), json!("2022-08-25T00:00:00Z"), json!("2022-08-25T00:00:00Z")],
@@ -3954,6 +4172,8 @@ mod tests {
Some("TIMESTAMP".to_string()),
],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![
json!("2022-08-25 09:58:43.123456"),
json!("2022-08-26T10:59:44Z"),
@@ -4002,6 +4222,8 @@ mod tests {
columns: vec!["created_at".to_string(), "recorded_at".to_string()],
column_types: vec![Some("timestamp".to_string()), Some("timestamp with time zone".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("2022-08-25T09:58:43.123456Z"), json!("2022-08-26T10:59:44+08:00")]],
batch_size: Some(100),
})
@@ -4027,6 +4249,8 @@ mod tests {
columns: vec!["enabled".to_string(), "mask".to_string(), "label".to_string()],
column_types: vec![Some("bit(1)".to_string()), Some("BIT(4)".to_string()), Some("varchar(20)".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("1"), json!("1010"), json!("1010")], vec![json!(false), json!(3), json!("off")]],
batch_size: Some(10),
})
@@ -4049,6 +4273,8 @@ mod tests {
columns: vec!["ENABLED".to_string(), "DELETED".to_string(), "OPTIONAL".to_string()],
column_types: vec![Some("BIT".to_string()), Some("bit".to_string()), Some("BIT".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(true), json!(false), Value::Null]],
batch_size: Some(10),
})
@@ -4078,6 +4304,8 @@ mod tests {
],
column_types: vec![Some("VARCHAR".to_string()); 6],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![
json!("plain"),
json!("eHall\0"),
@@ -4113,6 +4341,8 @@ mod tests {
columns: vec!["id".to_string(), "f_blob".to_string(), "note".to_string()],
column_types: vec![Some("int".to_string()), Some("blob".to_string()), Some("varchar(64)".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![json!("1"), json!("0x68656c6c6f"), json!("0x68656c6c6f")],
vec![json!("2"), json!("0X"), json!("1")],
@@ -4145,6 +4375,8 @@ mod tests {
Some("varchar(64)".to_string()),
],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![
json!(1),
json!("2026-06-12T10:11:12.123456789Z"),
@@ -4177,6 +4409,8 @@ mod tests {
Some("timestamp without time zone".to_string()),
],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("2026-06-12T10:11:12Z"), json!("2026-06-12T18:11:12+08:00")]],
batch_size: Some(10),
})
@@ -4201,6 +4435,8 @@ mod tests {
columns: vec!["row_version".to_string(), "created_at".to_string()],
column_types: vec![Some("timestamp".to_string()), Some("datetime2(3)".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!("2026-06-12T10:11:12Z"), json!("2026-06-12T10:11:12.1234567Z")]],
batch_size: Some(10),
})
@@ -4225,6 +4461,8 @@ mod tests {
columns: vec!["id".to_string(), "title".to_string(), "search_vector".to_string()],
column_types: vec![Some("integer".to_string()), Some("text".to_string()), Some("tsvector".to_string())],
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]],
batch_size: Some(10),
})
@@ -4271,6 +4509,8 @@ mod tests {
Some("stored generated".to_string()),
Some("DEFAULT_GENERATED".to_string()),
],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(7), json!(2), json!(3.5), json!(7.0), json!(7.0), json!("2026-07-30 08:00:00")]],
truncated: false,
}],
@@ -4336,6 +4576,8 @@ mod tests {
columns: vec!["ID".to_string(), "NAME".to_string()],
column_types: vec![Some("INT".to_string()), Some("VARCHAR(20)".to_string())],
column_extras: vec![Some("identity".to_string()), None],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("Ada")]],
batch_size: Some(10),
})
@@ -4365,6 +4607,8 @@ mod tests {
columns: vec!["id".to_string()],
column_types: Vec::new(),
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1)]],
truncated: true,
}],
@@ -4721,6 +4965,8 @@ mod tests {
columns: vec!["id".to_string(), "event_type".to_string()],
column_types: vec![None, None],
column_extras: vec![None, None],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("login")]],
batch_size: Some(100),
})
crates/dbx-core/src/query_result_export.rs
+22 -12
View File
@@ -24,6 +24,7 @@ use crate::query_result_sql::{
};
use crate::table_export::TableExportProgress;
use crate::transfer::keyset_pagination_sql;
use crate::types::SpatialColumn;
use crate::xlsx_export::{
finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook_with_options, StreamingXlsxWriter, XlsxWorksheetData,
};
@@ -297,8 +298,10 @@ struct SqlInsertWriter {
file: Option<BufWriter<File>>,
target: Option<StagedExportTarget>,
pending_rows: Vec<Vec<Value>>,
pending_spatial_values: Vec<Vec<Option<u32>>>,
columns: Vec<String>,
column_types: Vec<Option<String>>,
spatial_columns: Vec<SpatialColumn>,
database_type: DatabaseType,
schema: Option<String>,
table_name: String,
@@ -323,8 +326,10 @@ impl SqlInsertWriter {
file: Some(file),
target: Some(target),
pending_rows: Vec::new(),
pending_spatial_values: Vec::new(),
columns: Vec::new(),
column_types: Vec::new(),
spatial_columns: Vec::new(),
database_type: request.database_type,
schema: request.schema.clone(),
table_name,
@@ -339,14 +344,17 @@ impl SqlInsertWriter {
&mut self,
columns: Vec<String>,
result_column_types: &[String],
spatial_columns: &[SpatialColumn],
request: &QueryResultExportRequest,
) {
self.column_types = sql_insert_column_types(request, result_column_types);
self.spatial_columns = spatial_columns.to_vec();
self.columns = columns;
}
fn write_row(&mut self, row: Vec<Value>) -> Result<(), String> {
fn write_row(&mut self, row: Vec<Value>, spatial_values: Option<Vec<Option<u32>>>) -> Result<(), String> {
self.pending_rows.push(row);
self.pending_spatial_values.push(spatial_values.unwrap_or_default());
if self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
self.flush_batch()?;
}
@@ -366,6 +374,8 @@ impl SqlInsertWriter {
columns: self.columns.clone(),
column_types: self.column_types.clone(),
column_extras: Vec::new(),
spatial_columns: self.spatial_columns.clone(),
spatial_values: mem::take(&mut self.pending_spatial_values),
rows: mem::take(&mut self.pending_rows),
batch_size: Some(SQL_INSERT_BATCH_SIZE),
})?;
@@ -857,7 +867,7 @@ async fn export_query_result_core_inner(
columns = result.columns.clone();
column_types = result.column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, request);
writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request);
}
}
let fetched_row_count = result.rows.len();
@@ -886,8 +896,8 @@ async fn export_query_result_core_inner(
}
} else if format == "sql" {
let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?;
for row in formatted_rows.into_owned() {
writer.write_row(row)?;
for (row_index, row) in formatted_rows.into_owned().into_iter().enumerate() {
writer.write_row(row, result.spatial_values.get(row_index).cloned())?;
}
} else {
if xlsx.is_none() {
@@ -1060,7 +1070,7 @@ async fn try_export_postgres_query_result_stream(
columns = stream_columns;
temporal_column_types = column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, request);
writer.set_columns(columns.clone(), &column_types, &[], request);
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1082,7 +1092,7 @@ async fn try_export_postgres_query_result_stream(
request.date_time_format.as_deref(),
);
if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted.into_owned())?;
writer.write_row(formatted.into_owned(), None)?;
} else if let Some(file) = text_file.as_mut() {
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
} else if let Some(writer) = xlsx.as_mut() {
@@ -1304,7 +1314,7 @@ async fn try_export_mysql_query_result_stream(
columns = stream_columns;
temporal_column_types = column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, request);
writer.set_columns(columns.clone(), &column_types, &[], request);
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1326,7 +1336,7 @@ async fn try_export_mysql_query_result_stream(
request.date_time_format.as_deref(),
);
if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted.into_owned())?;
writer.write_row(formatted.into_owned(), None)?;
} else if let Some(file) = text_file.as_mut() {
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
} else if let Some(writer) = xlsx.as_mut() {
@@ -1519,7 +1529,7 @@ async fn try_export_clickhouse_query_result_stream(
columns = stream_columns;
temporal_column_types = column_types.clone();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &column_types, request);
writer.set_columns(columns.clone(), &column_types, &[], request);
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1541,7 +1551,7 @@ async fn try_export_clickhouse_query_result_stream(
request.date_time_format.as_deref(),
);
if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted.into_owned())?;
writer.write_row(formatted.into_owned(), None)?;
} else if let Some(file) = text_file.as_mut() {
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
} else if let Some(writer) = xlsx.as_mut() {
@@ -1703,7 +1713,7 @@ async fn try_export_sqlserver_query_result_stream(
columns = stream_columns.to_vec();
temporal_column_types = column_types.to_vec();
if let Some(writer) = sql_writer.as_mut() {
writer.set_columns(columns.clone(), &temporal_column_types, request);
writer.set_columns(columns.clone(), &temporal_column_types, &[], request);
} else if let Some(file) = text_file.as_mut() {
let header = format_text_export_header(format, &columns);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
@@ -1721,7 +1731,7 @@ async fn try_export_sqlserver_query_result_stream(
request.date_time_format.as_deref(),
);
if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted.into_owned())?;
writer.write_row(formatted.into_owned(), None)?;
} else if let Some(file) = text_file.as_mut() {
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
} else if let Some(writer) = xlsx.as_mut() {
crates/dbx-core/src/table_export.rs
+6
View File
@@ -1213,6 +1213,8 @@ async fn try_export_native_table_stream(
columns: col_names.to_vec(),
column_types: column_types.to_vec(),
column_extras: column_extras.to_vec(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: std::mem::take(pending_rows),
batch_size: Some(SQL_INSERT_BATCH_SIZE),
})?;
@@ -2022,6 +2024,8 @@ async fn export_table_data_core_inner(
columns: col_names.clone(),
column_types: column_types.clone(),
column_extras: column_extras.clone(),
spatial_columns: result.spatial_columns.clone(),
spatial_values: result.spatial_values.clone(),
rows: result.rows.clone(),
batch_size: Some(100),
})?;
@@ -2801,6 +2805,8 @@ mod tests {
columns,
column_types,
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![json!(1), json!("Ada")]],
batch_size: Some(100),
})
crates/dbx-core/tests/live_postgres_tsvector.rs
+2
View File
@@ -93,6 +93,8 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser
columns: result.columns.clone(),
column_types: result.column_types.iter().map(|value| Some(value.clone())).collect(),
column_extras: Vec::new(),
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: result.rows.clone(),
batch_size: Some(10),
})