From bac27394a1c69c20ec904729c593e59485c75c69 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 25 Nov 2014 12:21:22 -0500 Subject: Support arrays as input to array_agg() and ARRAY(SELECT ...). These cases formerly failed with errors about "could not find array type for data type". Now they yield arrays of the same element type and one higher dimension. The implementation involves creating functions with API similar to the existing accumArrayResult() family. I (tgl) also extended the base family by adding an initArrayResult() function, which allows callers to avoid special-casing the zero-inputs case if they just want an empty array as result. (Not all do, so the previous calling convention remains valid.) This allowed simplifying some existing code in xml.c and plperl.c. Ali Akbar, reviewed by Pavel Stehule, significantly modified by me --- src/backend/utils/adt/arrayfuncs.c | 495 ++++++++++++++++++++++++++++++++++--- 1 file changed, 460 insertions(+), 35 deletions(-) (limited to 'src/backend/utils/adt/arrayfuncs.c') diff --git a/src/backend/utils/adt/arrayfuncs.c b/src/backend/utils/adt/arrayfuncs.c index 6c8b41d2a91..743351b95e0 100644 --- a/src/backend/utils/adt/arrayfuncs.c +++ b/src/backend/utils/adt/arrayfuncs.c @@ -4573,10 +4573,58 @@ array_insert_slice(ArrayType *destArray, orignitems - orig_offset); } +/* + * initArrayResult - initialize an empty ArrayBuildState + * + * element_type is the array element type (must be a valid array element type) + * rcontext is where to keep working state + * + * Note: there are two common schemes for using accumArrayResult(). + * In the older scheme, you start with a NULL ArrayBuildState pointer, and + * call accumArrayResult once per element. In this scheme you end up with + * a NULL pointer if there were no elements, which you need to special-case. + * In the newer scheme, call initArrayResult and then call accumArrayResult + * once per element. In this scheme you always end with a non-NULL pointer + * that you can pass to makeArrayResult; you get an empty array if there + * were no elements. This is preferred if an empty array is what you want. + */ +ArrayBuildState * +initArrayResult(Oid element_type, MemoryContext rcontext) +{ + ArrayBuildState *astate; + MemoryContext arr_context; + + /* Make a temporary context to hold all the junk */ + arr_context = AllocSetContextCreate(rcontext, + "accumArrayResult", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + + astate = (ArrayBuildState *) + MemoryContextAlloc(arr_context, sizeof(ArrayBuildState)); + astate->mcontext = arr_context; + astate->alen = 64; /* arbitrary starting array size */ + astate->dvalues = (Datum *) + MemoryContextAlloc(arr_context, astate->alen * sizeof(Datum)); + astate->dnulls = (bool *) + MemoryContextAlloc(arr_context, astate->alen * sizeof(bool)); + astate->nelems = 0; + astate->element_type = element_type; + get_typlenbyvalalign(element_type, + &astate->typlen, + &astate->typbyval, + &astate->typalign); + + return astate; +} + /* * accumArrayResult - accumulate one (more) Datum for an array result * - * astate is working state (NULL on first call) + * astate is working state (can be NULL on first call) + * dvalue/disnull represent the new Datum to append to the array + * element_type is the Datum's type (must be a valid array element type) * rcontext is where to keep working state */ ArrayBuildState * @@ -4585,45 +4633,28 @@ accumArrayResult(ArrayBuildState *astate, Oid element_type, MemoryContext rcontext) { - MemoryContext arr_context, - oldcontext; + MemoryContext oldcontext; if (astate == NULL) { /* First time through --- initialize */ - - /* Make a temporary context to hold all the junk */ - arr_context = AllocSetContextCreate(rcontext, - "accumArrayResult", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); - oldcontext = MemoryContextSwitchTo(arr_context); - astate = (ArrayBuildState *) palloc(sizeof(ArrayBuildState)); - astate->mcontext = arr_context; - astate->alen = 64; /* arbitrary starting array size */ - astate->dvalues = (Datum *) palloc(astate->alen * sizeof(Datum)); - astate->dnulls = (bool *) palloc(astate->alen * sizeof(bool)); - astate->nelems = 0; - astate->element_type = element_type; - get_typlenbyvalalign(element_type, - &astate->typlen, - &astate->typbyval, - &astate->typalign); + astate = initArrayResult(element_type, rcontext); } else { - oldcontext = MemoryContextSwitchTo(astate->mcontext); Assert(astate->element_type == element_type); - /* enlarge dvalues[]/dnulls[] if needed */ - if (astate->nelems >= astate->alen) - { - astate->alen *= 2; - astate->dvalues = (Datum *) - repalloc(astate->dvalues, astate->alen * sizeof(Datum)); - astate->dnulls = (bool *) - repalloc(astate->dnulls, astate->alen * sizeof(bool)); - } + } + + oldcontext = MemoryContextSwitchTo(astate->mcontext); + + /* enlarge dvalues[]/dnulls[] if needed */ + if (astate->nelems >= astate->alen) + { + astate->alen *= 2; + astate->dvalues = (Datum *) + repalloc(astate->dvalues, astate->alen * sizeof(Datum)); + astate->dnulls = (bool *) + repalloc(astate->dnulls, astate->alen * sizeof(bool)); } /* @@ -4654,20 +4685,23 @@ accumArrayResult(ArrayBuildState *astate, /* * makeArrayResult - produce 1-D final result of accumArrayResult * - * astate is working state (not NULL) + * astate is working state (must not be NULL) * rcontext is where to construct result */ Datum makeArrayResult(ArrayBuildState *astate, MemoryContext rcontext) { + int ndims; int dims[1]; int lbs[1]; + /* If no elements were presented, we want to create an empty array */ + ndims = (astate->nelems > 0) ? 1 : 0; dims[0] = astate->nelems; lbs[0] = 1; - return makeMdArrayResult(astate, 1, dims, lbs, rcontext, true); + return makeMdArrayResult(astate, ndims, dims, lbs, rcontext, true); } /* @@ -4676,7 +4710,7 @@ makeArrayResult(ArrayBuildState *astate, * beware: no check that specified dimensions match the number of values * accumulated. * - * astate is working state (not NULL) + * astate is working state (must not be NULL) * rcontext is where to construct result * release is true if okay to release working state */ @@ -4713,6 +4747,397 @@ makeMdArrayResult(ArrayBuildState *astate, return PointerGetDatum(result); } +/* + * The following three functions provide essentially the same API as + * initArrayResult/accumArrayResult/makeArrayResult, but instead of accepting + * inputs that are array elements, they accept inputs that are arrays and + * produce an output array having N+1 dimensions. The inputs must all have + * identical dimensionality as well as element type. + */ + +/* + * initArrayResultArr - initialize an empty ArrayBuildStateArr + * + * array_type is the array type (must be a valid varlena array type) + * element_type is the type of the array's elements + * rcontext is where to keep working state + */ +ArrayBuildStateArr * +initArrayResultArr(Oid array_type, Oid element_type, MemoryContext rcontext) +{ + ArrayBuildStateArr *astate; + MemoryContext arr_context; + + /* Make a temporary context to hold all the junk */ + arr_context = AllocSetContextCreate(rcontext, + "accumArrayResultArr", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + + /* Note we initialize all fields to zero */ + astate = (ArrayBuildStateArr *) + MemoryContextAllocZero(arr_context, sizeof(ArrayBuildStateArr)); + astate->mcontext = arr_context; + + /* Save relevant datatype information */ + astate->array_type = array_type; + astate->element_type = element_type; + + return astate; +} + +/* + * accumArrayResultArr - accumulate one (more) sub-array for an array result + * + * astate is working state (can be NULL on first call) + * dvalue/disnull represent the new sub-array to append to the array + * array_type is the array type (must be a valid varlena array type) + * rcontext is where to keep working state + */ +ArrayBuildStateArr * +accumArrayResultArr(ArrayBuildStateArr *astate, + Datum dvalue, bool disnull, + Oid array_type, + MemoryContext rcontext) +{ + ArrayType *arg; + MemoryContext oldcontext; + int *dims, + *lbs, + ndims, + nitems, + ndatabytes; + char *data; + int i; + + /* + * We disallow accumulating null subarrays. Another plausible definition + * is to ignore them, but callers that want that can just skip calling + * this function. + */ + if (disnull) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("cannot accumulate null arrays"))); + + /* Detoast input array in caller's context */ + arg = DatumGetArrayTypeP(dvalue); + + if (astate == NULL) + { + /* First time through --- initialize */ + Oid element_type = get_element_type(array_type); + + if (!OidIsValid(element_type)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("data type %s is not an array type", + format_type_be(array_type)))); + astate = initArrayResultArr(array_type, element_type, rcontext); + } + else + { + Assert(astate->array_type == array_type); + } + + oldcontext = MemoryContextSwitchTo(astate->mcontext); + + /* Collect this input's dimensions */ + ndims = ARR_NDIM(arg); + dims = ARR_DIMS(arg); + lbs = ARR_LBOUND(arg); + data = ARR_DATA_PTR(arg); + nitems = ArrayGetNItems(ndims, dims); + ndatabytes = ARR_SIZE(arg) - ARR_DATA_OFFSET(arg); + + if (astate->ndims == 0) + { + /* First input; check/save the dimensionality info */ + + /* Should we allow empty inputs and just produce an empty output? */ + if (ndims == 0) + ereport(ERROR, + (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), + errmsg("cannot accumulate empty arrays"))); + if (ndims + 1 > MAXDIM) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("number of array dimensions (%d) exceeds the maximum allowed (%d)", + ndims + 1, MAXDIM))); + + /* + * The output array will have n+1 dimensions, with the ones after the + * first matching the input's dimensions. + */ + astate->ndims = ndims + 1; + astate->dims[0] = 0; + memcpy(&astate->dims[1], dims, ndims * sizeof(int)); + astate->lbs[0] = 1; + memcpy(&astate->lbs[1], lbs, ndims * sizeof(int)); + + /* Allocate at least enough data space for this item */ + astate->abytes = 1024; + while (astate->abytes <= ndatabytes) + astate->abytes *= 2; + astate->data = (char *) palloc(astate->abytes); + } + else + { + /* Second or later input: must match first input's dimensionality */ + if (astate->ndims != ndims + 1) + ereport(ERROR, + (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), + errmsg("cannot accumulate arrays of different dimensionality"))); + for (i = 0; i < ndims; i++) + { + if (astate->dims[i + 1] != dims[i] || astate->lbs[i + 1] != lbs[i]) + ereport(ERROR, + (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), + errmsg("cannot accumulate arrays of different dimensionality"))); + } + + /* Enlarge data space if needed */ + if (astate->nbytes + ndatabytes > astate->abytes) + { + astate->abytes = Max(astate->abytes * 2, + astate->nbytes + ndatabytes); + astate->data = (char *) repalloc(astate->data, astate->abytes); + } + } + + /* + * Copy the data portion of the sub-array. Note we assume that the + * advertised data length of the sub-array is properly aligned. We do not + * have to worry about detoasting elements since whatever's in the + * sub-array should be OK already. + */ + memcpy(astate->data + astate->nbytes, data, ndatabytes); + astate->nbytes += ndatabytes; + + /* Deal with null bitmap if needed */ + if (astate->nullbitmap || ARR_HASNULL(arg)) + { + int newnitems = astate->nitems + nitems; + + if (astate->nullbitmap == NULL) + { + /* + * First input with nulls; we must retrospectively handle any + * previous inputs by marking all their items non-null. + */ + astate->aitems = 256; + while (astate->aitems <= newnitems) + astate->aitems *= 2; + astate->nullbitmap = (bits8 *) palloc((astate->aitems + 7) / 8); + array_bitmap_copy(astate->nullbitmap, 0, + NULL, 0, + astate->nitems); + } + else if (newnitems > astate->aitems) + { + astate->aitems = Max(astate->aitems * 2, newnitems); + astate->nullbitmap = (bits8 *) + repalloc(astate->nullbitmap, (astate->aitems + 7) / 8); + } + array_bitmap_copy(astate->nullbitmap, astate->nitems, + ARR_NULLBITMAP(arg), 0, + nitems); + } + + astate->nitems += nitems; + astate->dims[0] += 1; + + MemoryContextSwitchTo(oldcontext); + + /* Release detoasted copy if any */ + if ((Pointer) arg != DatumGetPointer(dvalue)) + pfree(arg); + + return astate; +} + +/* + * makeArrayResultArr - produce N+1-D final result of accumArrayResultArr + * + * astate is working state (must not be NULL) + * rcontext is where to construct result + * release is true if okay to release working state + */ +Datum +makeArrayResultArr(ArrayBuildStateArr *astate, + MemoryContext rcontext, + bool release) +{ + ArrayType *result; + MemoryContext oldcontext; + + /* Build the final array result in rcontext */ + oldcontext = MemoryContextSwitchTo(rcontext); + + if (astate->ndims == 0) + { + /* No inputs, return empty array */ + result = construct_empty_array(astate->element_type); + } + else + { + int dataoffset, + nbytes; + + /* Compute required space */ + nbytes = astate->nbytes; + if (astate->nullbitmap != NULL) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(astate->ndims, astate->nitems); + nbytes += dataoffset; + } + else + { + dataoffset = 0; + nbytes += ARR_OVERHEAD_NONULLS(astate->ndims); + } + + result = (ArrayType *) palloc0(nbytes); + SET_VARSIZE(result, nbytes); + result->ndim = astate->ndims; + result->dataoffset = dataoffset; + result->elemtype = astate->element_type; + + memcpy(ARR_DIMS(result), astate->dims, astate->ndims * sizeof(int)); + memcpy(ARR_LBOUND(result), astate->lbs, astate->ndims * sizeof(int)); + memcpy(ARR_DATA_PTR(result), astate->data, astate->nbytes); + + if (astate->nullbitmap != NULL) + array_bitmap_copy(ARR_NULLBITMAP(result), 0, + astate->nullbitmap, 0, + astate->nitems); + } + + MemoryContextSwitchTo(oldcontext); + + /* Clean up all the junk */ + if (release) + MemoryContextDelete(astate->mcontext); + + return PointerGetDatum(result); +} + +/* + * The following three functions provide essentially the same API as + * initArrayResult/accumArrayResult/makeArrayResult, but can accept either + * scalar or array inputs, invoking the appropriate set of functions above. + */ + +/* + * initArrayResultAny - initialize an empty ArrayBuildStateAny + * + * input_type is the input datatype (either element or array type) + * rcontext is where to keep working state + */ +ArrayBuildStateAny * +initArrayResultAny(Oid input_type, MemoryContext rcontext) +{ + ArrayBuildStateAny *astate; + Oid element_type = get_element_type(input_type); + + if (OidIsValid(element_type)) + { + /* Array case */ + ArrayBuildStateArr *arraystate; + + arraystate = initArrayResultArr(input_type, element_type, rcontext); + astate = (ArrayBuildStateAny *) + MemoryContextAlloc(arraystate->mcontext, + sizeof(ArrayBuildStateAny)); + astate->scalarstate = NULL; + astate->arraystate = arraystate; + } + else + { + /* Scalar case */ + ArrayBuildState *scalarstate; + + /* Let's just check that we have a type that can be put into arrays */ + Assert(OidIsValid(get_array_type(input_type))); + + scalarstate = initArrayResult(input_type, rcontext); + astate = (ArrayBuildStateAny *) + MemoryContextAlloc(scalarstate->mcontext, + sizeof(ArrayBuildStateAny)); + astate->scalarstate = scalarstate; + astate->arraystate = NULL; + } + + return astate; +} + +/* + * accumArrayResultAny - accumulate one (more) input for an array result + * + * astate is working state (can be NULL on first call) + * dvalue/disnull represent the new input to append to the array + * input_type is the input datatype (either element or array type) + * rcontext is where to keep working state + */ +ArrayBuildStateAny * +accumArrayResultAny(ArrayBuildStateAny *astate, + Datum dvalue, bool disnull, + Oid input_type, + MemoryContext rcontext) +{ + if (astate == NULL) + astate = initArrayResultAny(input_type, rcontext); + + if (astate->scalarstate) + (void) accumArrayResult(astate->scalarstate, + dvalue, disnull, + input_type, rcontext); + else + (void) accumArrayResultArr(astate->arraystate, + dvalue, disnull, + input_type, rcontext); + + return astate; +} + +/* + * makeArrayResultAny - produce final result of accumArrayResultAny + * + * astate is working state (must not be NULL) + * rcontext is where to construct result + * release is true if okay to release working state + */ +Datum +makeArrayResultAny(ArrayBuildStateAny *astate, + MemoryContext rcontext, bool release) +{ + Datum result; + + if (astate->scalarstate) + { + /* Must use makeMdArrayResult to support "release" parameter */ + int ndims; + int dims[1]; + int lbs[1]; + + /* If no elements were presented, we want to create an empty array */ + ndims = (astate->scalarstate->nelems > 0) ? 1 : 0; + dims[0] = astate->scalarstate->nelems; + lbs[0] = 1; + + result = makeMdArrayResult(astate->scalarstate, ndims, dims, lbs, + rcontext, release); + } + else + { + result = makeArrayResultArr(astate->arraystate, + rcontext, release); + } + return result; +} + + Datum array_larger(PG_FUNCTION_ARGS) { -- cgit v1.2.3