Sponsor
DSL References
DSL stands for Domain Specific Language. In Flow, the DSL is a set of small functions that wrap object construction so pipelines read top-to-bottom. See the examples for usage in context.
SCHEMA
bool_schema
bool_schema(string $name, bool $nullable, ?Metadata $metadata) : BooleanDefinitiondatetime_schema(string $name, bool $nullable, ?Metadata $metadata) : DateTimeDefinitiondate_schema(string $name, bool $nullable, ?Metadata $metadata) : DateDefinition/**
* Create a Definition from an array representation.
*
* @param array<array-key, mixed> $definition
*
* @return Definition<mixed>
*/
definition_from_array(array $definition) : Definition/**
* Create a Definition from a Type.
*
* @param Type<mixed> $type
*
* @return Definition<mixed>
*/
definition_from_type(Reference|string $ref, Type $type, bool $nullable, ?Metadata $metadata) : Definition/**
* @template T of \UnitEnum
*
* @param class-string<T> $type
*
* @return EnumDefinition<T>
*/
enum_schema(string $name, string $type, bool $nullable, ?Metadata $metadata) : EnumDefinitionfloat_schema(string $name, bool $nullable, ?Metadata $metadata) : FloatDefinitionhtml_element_schema(string $name, bool $nullable, ?Metadata $metadata) : HTMLElementDefinitionhtml_schema(string $name, bool $nullable, ?Metadata $metadata) : HTMLDefinitioninfer_schema() : SchemaInferenceBuilderinteger_schema(string $name, bool $nullable, ?Metadata $metadata) : IntegerDefinition/**
* Alias for `integer_schema`.
*/
int_schema(string $name, bool $nullable, ?Metadata $metadata) : IntegerDefinitionjson_schema(string $name, bool $nullable, ?Metadata $metadata) : JsonDefinition/**
* @template T
*
* @param ListType<list<T>>|Type<list<T>> $type
*
* @return ListDefinition<T>
*/
list_schema(string $name, ListType|Type $type, bool $nullable, ?Metadata $metadata) : ListDefinition/**
* @template TKey of array-key
* @template TValue
*
* @param MapType<array<TKey, TValue>>|Type<array<TKey, TValue>> $type
*
* @return MapDefinition<TKey, TValue>
*/
map_schema(string $name, MapType|Type $type, bool $nullable, ?Metadata $metadata) : MapDefinitionnull_schema(string $name, ?Metadata $metadata) : NullDefinition/**
* @param Definition<mixed> ...$definitions
*
* @return Schema
*/
schema(Definition $definitions) : Schemaschema_sort_by_metadata(string $key, SortOrder $order) : SortingStrategyschema_sort_by_name(SortOrder $order) : SortingStrategy/**
* @param array<class-string<Type<mixed>>, int> $priorities
*/
schema_sort_by_type(array $priorities, SortOrder $order) : SortingStrategy/**
* @param array<class-string<Type<mixed>>, int> $priorities
*/
schema_sort_by_type_and_name(array $priorities, SortOrder $order) : SortingStrategystring_schema(string $name, bool $nullable, ?Metadata $metadata) : StringDefinition/**
* @template T
*
* @param StructureType<array<array-key, T>>|Type<array<array-key, T>> $type
*
* @return StructureDefinition<T>
*/
structure_schema(string $name, StructureType|Type $type, bool $nullable, ?Metadata $metadata) : StructureDefinition/**
* Alias for `string_schema`.
*/
str_schema(string $name, bool $nullable, ?Metadata $metadata) : StringDefinitiontime_schema(string $name, bool $nullable, ?Metadata $metadata) : TimeDefinitiontime_zone_schema(string $name, bool $nullable, ?Metadata $metadata) : TimeZoneDefinition/**
* @param Type<mixed>|UnionType<mixed, mixed> $type
*
* @deprecated a column holds exactly one type - use definition_from_type() instead
*
* @return Definition<mixed>
*/
union_schema(string $name, UnionType|Type $type, bool $nullable, ?Metadata $metadata) : Definitionuuid_schema(string $name, bool $nullable, ?Metadata $metadata) : UuidDefinitionxml_element_schema(string $name, bool $nullable, ?Metadata $metadata) : XMLElementDefinitionxml_schema(string $name, bool $nullable, ?Metadata $metadata) : XMLDefinitionDATA_FRAME
/**
* Alias for save_mode_append().
*/
append() : SaveMode/**
* @param array<array<mixed>>|array<mixed|string> $data
* @param array<Partition>|Partitions $partitions
*/
array_to_row(array $data, Schema $schema, Hydrator $hydrator, Partitions|array $partitions) : Row/**
* @param array<array<mixed>>|array<mixed|string> $data
*/
array_to_rows(array $data, Schema $schema, Hydrator $hydrator) : Rows/**
* An alias for `ref`.
*/
col(string $entry) : UnresolvedReferenceconfig() : Configconfig_builder() : ConfigBuildercurrent_row() : FrameBounddata_frame(Config|ConfigBuilder|null $config) : Flow/**
* Alias for data_frame() : Flow.
*/
df(Config|ConfigBuilder|null $config) : Flow/**
* Alias for save_mode_exception_if_exists().
*/
exception_if_exists() : SaveModeexecution_context(?Config $config) : FlowContextexternal_sort() : ExternalSortBuilderfilesystem_cache(Path|string|null $cache_dir, Filesystem $filesystem, Serializer $serializer) : FilesystemCacheflow_context(?Config $config) : FlowContextfollowing(int $offset) : FrameBoundgenerate_random_int(int $start, int $end, RandomValueGenerator $generator) : intgenerate_random_string(int $length, RandomValueGenerator $generator) : stringhash_group_by() : HashGroupByBuilderhash_join() : HashJoinBuilderhash_repartition() : HashRepartitionBuilder/**
* Alias for save_mode_ignore().
*/
ignore() : SaveModeignore_error_handler() : IgnoreError/**
* @param array<string|Type<mixed>>|Type<mixed> $type
* @param mixed $value
*/
is_type(Type|array $type, ?mixed $value) : bool/**
* @param array<Comparison|string>|Comparison $comparisons
*/
join_on(Comparison|array $comparisons, string $join_prefix) : Expressionmemory_sort() : MemorySortBuilder/**
* Alias for save_mode_overwrite().
*/
overwrite() : SaveModepreceding(int $offset) : FrameBoundprint_rows(Rows $rows, int|bool $truncate, ?Formatter $formatter) : stringrandom_string(ScalarFunction|int $length, RandomValueGenerator $generator) : RandomString/**
* @param array<array-key, mixed> $values
*/
row(array $values) : Rowrows(Schema $schema, Row $row) : Rowsrow_number() : RowNumbersave_mode_append() : SaveModesave_mode_exception_if_exists() : SaveModesave_mode_ignore() : SaveModesave_mode_overwrite() : SaveModeskip_rows_handler() : SkipRowsthrow_error_handler() : ThrowErrorunbounded_following() : FrameBoundunbounded_preceding() : FrameBoundwindow() : WindowEXTRACTOR
/**
* @param null|int<1, max> $min_size
*/
batched_by(Extractor $extractor, Reference|string $column, ?int $min_size) : BatchByExtractor/**
* @param int<1, max> $size
*/
batches(Extractor $extractor, int $size) : BatchExtractorfiles(Path|string $directory, Filesystem $filesystem) : FilesExtractorfrom_all(Extractor $extractors) : ChainExtractor/**
* @param iterable<array<mixed>> $array
* @param null|Schema $schema - @deprecated use withSchema() method instead
* @param null|Path $spillRoot - where a non-array $array is spilled while it is described; null resolves to
* $filesystem->getSystemTmpDir() and only on that path
*/
from_array(iterable $array, ?Schema $schema, Filesystem $filesystem, ?Path $spillRoot) : ArrayExtractor/**
* @param string $id - cache id from which data will be extracted
* @param null|Extractor $fallback_extractor - extractor that will be used when cache is empty - @deprecated use withFallbackExtractor() method instead
* @param bool $clear - clear cache after extraction - @deprecated use withClearOnFinish() method instead
*/
from_cache(string $id, ?Extractor $fallback_extractor, bool $clear, ?Cache $cache) : CacheExtractorfrom_data_frame(DataFrame $data_frame) : DataFrameExtractorfrom_memory(Memory $memory) : MemoryExtractorfrom_path_partitions(Path|string $path, Filesystem $filesystem) : PathPartitionsExtractorfrom_rows(Rows $rows) : RowsExtractorfrom_sequence_date_period(string $entry_name, DateTimeInterface $start, DateInterval $interval, DateTimeInterface $end, int $options) : SequenceExtractorfrom_sequence_date_period_recurrences(string $entry_name, DateTimeInterface $start, DateInterval $interval, int $recurrences, int $options) : SequenceExtractorfrom_sequence_number(string $entry_name, string|int|float $start, string|int|float $end, int|float $step) : SequenceExtractor/**
* @param Type<mixed> ...$types partition column name => type, passed as named arguments
*/
partition_types(Type $types) : PartitionTypesTRANSFORMER
add_row_index(string $column, StartFrom $startFrom) : AddRowIndex/**
* @param int<1, max> $size
*/
batch_size(int $size) : BatchSizedrop(Reference|string $entries) : Droplimit(?int $limit) : Limit/**
* @param array<int, string> $columns
*/
mask_columns(array $columns, string $mask) : MaskColumns/**
* @param array<array-key, string> $renames Map of old_name => new_name
*/
rename_map(array $renames) : RenameMapEntryStrategy/**
* @param array<string>|string $search
* @param array<string>|string $replace
*/
rename_replace(array|string $search, array|string $replace) : RenameReplaceEntryStrategyrename_style(StringStyles $style) : RenameCaseEntryStrategyselect(Reference|string $entries) : SelectAGGREGATING_FUNCTION
average(Reference|string $ref, int $scale, Rounding $rounding) : Averagecollect(Reference|string $ref) : Collectcollect_unique(Reference|string $ref) : CollectUniquecount(?Reference $function) : Countfirst(Reference|string $ref) : Firstlast(Reference|string $ref) : Lastmax(Reference|string $ref) : Maxmin(Reference|string $ref) : Minstring_agg(Reference|string $ref, string $separator, ?SortOrder $sort) : StringAggregatesum(Reference|string $ref, ScalarFunction|bool $exact) : SumSCALAR_FUNCTION
all(ScalarFunction $functions) : Allany(ScalarFunction $values) : Any/**
* @param array<array-key, mixed>|ScalarFunction $ref
*/
array_exists(ScalarFunction|array $ref, ScalarFunction|string $path) : ArrayPathExists/**
* Expands each value into entry, if there are more than one value, multiple rows will be created.
* Array keys are ignored, only values are used to create new rows.
* Nested in another function (structure(), concat(), ...) it still gives one row per element. Several
* expands in one expression are zipped to the longest list; a shorter one gives null, so its element
* type becomes nullable. It is refused inside another array_expand() and in filter(), until(),
* duplicateRow(), aggregate(), over() and onEach().
*
* Before:
* +--+-------------------+
* |id| array|
* +--+-------------------+
* | 1|{"a":1,"b":2,"c":3}|
* +--+-------------------+
*
* After:
* +--+--------+
* |id|expanded|
* +--+--------+
* | 1| 1|
* | 1| 2|
* | 1| 3|
* +--+--------+
*/
array_expand(ScalarFunction $function, ArrayExpand $expand) : ArrayExpandarray_get(ScalarFunction $ref, string $path) : ArrayGet/**
* @param array<array-key, mixed>|ScalarFunction $keys
*/
array_get_collection(ScalarFunction $ref, ScalarFunction|array $keys) : ArrayGetCollectionarray_get_collection_first(ScalarFunction $ref, string $keys) : ArrayGetCollectionarray_keys_style_convert(ScalarFunction $ref, StringStyles|string $style) : ArrayKeysStyleConvertarray_key_rename(ScalarFunction $ref, string $path, string $newName) : ArrayKeyRename/**
* @param array<array-key, mixed>|ScalarFunction $left
* @param array<array-key, mixed>|ScalarFunction $right
*/
array_merge(ScalarFunction|array $left, ScalarFunction|array $right) : ArrayMerge/**
* @param array<array-key, mixed>|ScalarFunction $array
*/
array_merge_collection(ScalarFunction|array $array) : ArrayMergeCollection/**
* @param array<array-key, mixed>|ScalarFunction $function
*/
array_reverse(ScalarFunction|array $function, bool $preserveKeys) : ArrayReversearray_sort(ScalarFunction $function, ?Sort $sort_function, ScalarFunction|int|null $flags, ScalarFunction|bool $recursive) : ArraySort/**
* Unpacks each element of an array into a new entry, using the array key as the entry name.
*
* Before:
* +--+-------------------+
* |id| array|
* +--+-------------------+
* | 1|{"a":1,"b":2,"c":3}|
* | 2|{"d":4,"e":5,"f":6}|
* +--+-------------------+
*
* After:
* +--+-----+-----+-----+-----+-----+
* |id|arr.b|arr.c|arr.d|arr.e|arr.f|
* +--+-----+-----+-----+-----+-----+
* | 1| 2| 3| | | |
* | 2| | | 4| 5| 6|
* +--+-----+-----+-----+-----+-----+
*
* @param array<array-key, mixed>|ScalarFunction $array
*/
array_unpack(ScalarFunction|array $array, Schema $schema) : ArrayUnpackbetween(?mixed $value, ?mixed $lower_bound, ?mixed $upper_bound, ScalarFunction|Boundary $boundary) : Between/**
* Calls a user-defined function with the given parameters.
*
* @param Type<mixed> $return_type
* @param array<mixed> $parameters
*/
call(ScalarFunction $callable, Type $return_type, array $parameters) : CallUserFunccapitalize(ScalarFunction|string $value) : Capitalize/**
* @param \Flow\Types\Type<mixed>|string $type
*/
cast(?mixed $value, Type|string $type) : Castcoalesce(ScalarFunction $values) : Coalesce/**
* @param array<array-key, mixed>|ScalarFunction $keys
* @param array<array-key, mixed>|ScalarFunction $values
*/
combine(ScalarFunction|array $keys, ScalarFunction|array $values) : Combine/**
* Concat all values. If you want to concatenate values with separator use concat_ws function.
*/
concat(ScalarFunction|string $functions) : Concat/**
* Concat all values with separator.
*/
concat_ws(ScalarFunction|string $separator, ScalarFunction|string $functions) : ConcatWithSeparatordate_time_format(ScalarFunction $ref, string $format) : DateTimeFormat/**
* An alias for `ref`.
*/
entry(string $entry) : UnresolvedReferenceenum_name(?mixed $value) : EnumNameenum_value(?mixed $value) : EnumValueexists(ScalarFunction $ref) : Existsgreatest(?mixed $values) : Greatesthash(?mixed $value, Algorithm $algorithm) : Hashleast(?mixed $values) : Leastlist_ref(string $entry) : ListFunctionslit(?mixed $value) : Literallower(ScalarFunction|string $value) : ToLower/**
* @param array<MatchCondition> $cases
*/
match_cases(array $cases, ?mixed $default) : MatchCasesmatch_condition(?mixed $condition, ?mixed $then) : MatchConditionnot(ScalarFunction $value) : Notnow(DateTimeZone|ScalarFunction $time_zone) : Nownumber_format(ScalarFunction|int|float $value, ScalarFunction|int $decimals, ScalarFunction|string $decimal_separator, ScalarFunction|string $thousands_separator) : NumberFormatoptional(ScalarFunction $function) : Optionalref(string $entry) : UnresolvedReferencerefs(Reference|string $entries) : Referencesregex(ScalarFunction|string $pattern, ScalarFunction|string $subject, int $flags, ScalarFunction|int $offset) : Regexregex_all(ScalarFunction|string $pattern, ScalarFunction|string $subject, int $flags, ScalarFunction|int $offset) : RegexAllregex_match(ScalarFunction|string $pattern, ScalarFunction|string $subject, ScalarFunction|int $flags, ScalarFunction|int $offset) : RegexMatchregex_match_all(ScalarFunction|string $pattern, ScalarFunction|string $subject, ScalarFunction|int $flags, ScalarFunction|int $offset) : RegexMatchAllregex_replace(ScalarFunction|string $pattern, ScalarFunction|string $replacement, ScalarFunction|string $subject, ScalarFunction|int|null $limit) : RegexReplaceround(ScalarFunction|int|float $value, ScalarFunction|int $precision, ScalarFunction|int $mode) : Roundsanitize(ScalarFunction|string $value, ScalarFunction|string $placeholder, ScalarFunction|int|null $skipCharacters) : Sanitizesize(?mixed $value) : Sizesplit(ScalarFunction|string $value, ScalarFunction|string $separator, ScalarFunction|int $limit) : Splitsprintf(ScalarFunction|string $format, ScalarFunction|string|int|float|null $args) : Sprintf/**
* Builds a structure from scalar functions: one element per key, in key order.
* An element is nullable when its function is; the structure itself never is.
*
* @param array<array-key, ScalarFunction> $elements
*/
structure(array $elements) : Structure/**
* Alias for `array_get`.
*/
structure_get(ScalarFunction $ref, string $path) : ArrayGet/**
* Alias for `array_get_collection`.
*
* @param array<array-key, mixed>|ScalarFunction $keys
*/
structure_get_collection(ScalarFunction $ref, ScalarFunction|array $keys) : ArrayGetCollection/**
* Alias for `array_get_collection_first`.
*/
structure_get_collection_first(ScalarFunction $ref, string $keys) : ArrayGetCollectionstructure_ref(string $entry) : StructureFunctionsto_date(?mixed $ref, ScalarFunction|string $format, ScalarFunction|DateTimeZone $timeZone) : ToDateto_date_time(?mixed $ref, ScalarFunction|string $format, ScalarFunction|DateTimeZone $timeZone) : ToDateTimeto_timezone(ScalarFunction|DateTimeInterface $value, ScalarFunction|DateTimeZone|string $timeZone) : ToTimeZoneulid(ScalarFunction|string|null $value) : Ulidupper(ScalarFunction|string $value) : ToUpperuuid_v4() : Uuiduuid_v7(ScalarFunction|DateTimeInterface $value) : Uuidwhen(?mixed $condition, ?mixed $then, ?mixed $else) : WhenWINDOW_FUNCTION
COMPARISON
compare_all(Comparison $comparison, Comparison $comparisons) : Allcompare_any(Comparison $comparison, Comparison $comparisons) : Anyequal(Reference|string $left, Reference|string $right) : Equalidentical(Reference|string $left, Reference|string $right) : IdenticalHELPER
analyze() : Analyzeclock(string $time_zone) : ClockInterfaceconstraint_sorted_by(Reference|string $column, Reference|string $columns) : SortedByConstraintconstraint_unique(string $reference, string $references) : UniqueConstraintdate_interval_to_microseconds(DateInterval $interval) : intdate_interval_to_milliseconds(DateInterval $interval) : intdate_interval_to_seconds(DateInterval $interval) : intdelay_exponential(Duration $base, int $multiplier, ?Duration $max_delay) : Exponentialdelay_fixed(Duration $delay) : Fixed/**
* @param float $jitter_factor a value between 0 and 1 representing the maximum percentage of jitter to apply
*/
delay_jitter(DelayFactory $delay, float $jitter_factor) : Jitterdelay_linear(Duration $delay, Duration $increment) : Linear/**
* Reads the pivot column once at build time and turns what it finds into declared values. Refuses a
* source that cannot be read twice.
*/
discover_pivot_values(int $maxValues) : DiscoveredPivotValuesduration_microseconds(int $microseconds) : Durationduration_milliseconds(int $milliseconds) : Durationduration_minutes(int $minutes) : Durationduration_seconds(int $seconds) : Duration/**
* Declares the pivot columns a groupBy()->pivot() produces, so the plan can name them before a row flows.
*/
pivot_values(string|int $values) : DeclaredPivotValuesretry_any_throwable(int $limit) : AnyThrowable/**
* @param array<class-string<\Throwable>> $exception_types
*/
retry_any_throwable_except(array $exception_types, int $limit) : AnyThrowableExcept/**
* @param array<class-string<\Throwable>> $exception_types
*/
retry_on_exception_types(array $exception_types, int $limit) : OnExceptionTypesschema_evolving_validator() : EvolvingValidator/**
* @return Schema
*/
schema_from_json(string $schema) : Schema/**
* @param array<string, array<bool|float|int|string>|bool|float|int|string> $metadata
*/
schema_metadata(array $metadata) : Metadataschema_selective_validator() : SelectiveValidatorschema_strict_validator() : StrictValidator/**
* @param Schema $schema
*/
schema_to_ascii(Schema $schema, ?SchemaFormatter $formatter) : string/**
* @param Schema $schema
*/
schema_to_json(Schema $schema, bool $pretty) : string/**
* @param Schema $schema
*/
schema_to_php(Schema $schema, ValueFormatter $valueFormatter, TypeFormatter $typeFormatter) : string/**
* @param Schema $expected
* @param Schema $given
*/
schema_validate(Schema $expected, Schema $given, SchemaValidator $validator) : ValidationContextserialize_to_string(Serializer $serializer, Rows $rows) : stringtelemetry_options(bool $trace_loading, bool $trace_transformations, bool $trace_cache, bool $collect_metrics) : TelemetryOptionsunserialize_from_string(Serializer $serializer, string $payload) : Rowswith_entry(string $name, ScalarFunction $function) : WithEntryLOADER
partition_by(Reference|string $entry, Reference|string $entries) : Partitioning/**
* Convert rows to an array and store them in passed array variable.
*
* @param array<array-key, mixed> $array
*
* @param-out array<array<mixed>> $array
*/
to_array(array $array) : ArrayLoaderto_branch(ScalarFunction $condition, Loader $loader, ?Transformation $transformation) : BranchingLoaderto_memory(Memory $memory) : MemoryLoaderto_output(int|bool $truncate, Output $output, Formatter $formatter, SchemaFormatter $schemaFormatter) : StreamLoaderto_stderr(int|bool $truncate, Output $output, Formatter $formatter, SchemaFormatter $schemaFormatter) : StreamLoaderto_stdout(int|bool $truncate, Output $output, Formatter $formatter, SchemaFormatter $schemaFormatter) : StreamLoaderto_stream(string $uri, int|bool $truncate, Output $output, string $mode, Formatter $formatter, SchemaFormatter $schemaFormatter) : StreamLoaderto_transformation(Transformer|Transformation $transformer, Loader $loader) : TransformerLoaderwrite_with_retries(Loader $loader, RetryStrategy $retry_strategy, DelayFactory $delay_factory, Sleep $sleep) : RetryLoader
