From 741906fad6407e51a11a271ab8a8f28d4c3aa53c Mon Sep 17 00:00:00 2001 From: Dan Harrin Date: Sun, 5 Nov 2023 12:32:44 +0000 Subject: [PATCH] wip --- .../docs/07-prebuilt-actions/08-import.md | 575 +++++++++++++++++- packages/actions/src/ImportAction.php | 4 +- packages/actions/src/Imports/ImportColumn.php | 65 +- packages/actions/src/Imports/Importer.php | 14 +- packages/forms/docs/05-validation.md | 2 +- .../docs/05-customizing-notifications.md | 2 +- 6 files changed, 640 insertions(+), 22 deletions(-) diff --git a/packages/actions/docs/07-prebuilt-actions/08-import.md b/packages/actions/docs/07-prebuilt-actions/08-import.md index 6e9c78d3f0..b9d3998804 100644 --- a/packages/actions/docs/07-prebuilt-actions/08-import.md +++ b/packages/actions/docs/07-prebuilt-actions/08-import.md @@ -8,13 +8,584 @@ Filament includes a prebuilt action that is able to import rows from a CSV. When ```php use App\Filament\Imports\ProductImporter; -use Filament\Actions\ImportAction; ImportAction::make() ->importer(ProductImporter::class) ``` -The "importer" class needs to be created to tell Filament how to import each row of the CSV. +The ["importer" class needs to be created](#creating-an-importer) to tell Filament how to import each row of the CSV. ## Creating an importer +To create an importer class for a model, you may use the `make:filament-importer` command, passing the name of a model: + +```bash +php artisan make:filament-importer Product +``` + +This will create a new class in the `app/Filament/Imports` directory. You now need to define the [columns](#defining-importer-columns) that can be imported. + +### Automatically generating importer columns + +If you'd like to save time, Filament can automatically generate the [columns](#defining-importer-columns) for you, based on your model's database columns, using `--generate`: + +```bash +php artisan make:filament-importer Product --generate +``` + +> If your table contains ENUM columns, the `doctrine/dbal` package we use is unable to scan your table and will crash. Hence, Filament is unable to generate the columns for your importer if it contains an ENUM column. Read more about this issue [here](https://github.com/doctrine/dbal/issues/3819#issuecomment-573419808). + +## Defining importer columns + +To define the columns that can be imported, you need to override the `getColumns()` method on your importer class, returning an array of `ImportColumn` objects: + +```php +use Filament\Actions\Imports\ImportColumn; + +public function getColumns(): array +{ + return [ + ImportColumn::make('name') + ->requiredMapping() + ->rules(['required', 'max:255']), + ImportColumn::make('sku') + ->label('SKU') + ->requiredMapping() + ->rules(['required', 'max:32']), + ImportColumn::make('price') + ->numeric() + ->rules(['numeric', 'min:0']), + ]; +} +``` + +### Customizing the label of an import column + +The label for each column will be generated automatically from its name, but you can override it by calling the `label()` method: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('sku') + ->label('SKU') +``` + +### Requiring an importer column to be mapped to a CSV column + +You can call the `requiredMapping()` method to make a column required to be mapped to a column in the CSV. Columns that are required in the database should be required to be mapped: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('sku') + ->requiredMapping() +``` + +If you require a column in the database, you also need to make sure that it has a [`rules(['required'])` validation rule](#validating-csv-data). + +### Validating CSV data + +You can call the `rules()` method to add validation rules to a column. These rules will check the data in reach row from the CSV before it is saved to the database: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('sku') + ->rules(['required', 'max:32']) +``` + +Any rows that do not pass validation will not be imported. Instead, they will be compiled into a new CSV of "failed rows", which the user can download after the import has finished. The user will be shown a list of validation errors for each row that failed. + +### Casting state + +Before [validation](#validating-csv-data), data from the CSV can be cast. This is useful for converting strings into the correct data type, otherwise validation may fail. For example, if you have a `price` column in your CSV, you may want to cast it to a float: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('price') + ->castStateUsing(function (string $state): ?float { + if (blank($state)) { + return null; + } + + $state = preg_replace('/[^0-9.]/', '', $state); + $state = floatval($state); + + return round($state, precision: 2); + }) +``` + +In this example, we pass in a function that is used to cast the `$state`. This function removes any non-numeric characters from the string, casts it to a float, and rounds it to two decimal places. + +> Please note: if a column is not [required by validation](#validating-csv-data), and it is empty, it will not be cast. + +Filament also ships with some built-in casting methods: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('price') + ->numeric() // Casts the state to a float. + +ImportColumn::make('price') + ->numeric(decimalPlaces: 2) // Casts the state to a float, and rounds it to 2 decimal places. + +ImportColumn::make('quantity') + ->integer() // Casts the state to an integer. + +ImportColumn::make('is_visible') + ->boolean() // Casts the state to a boolean. +``` + +#### Mutating the state after it has been cast + +If you're using a [built-in casting method](#casting-state) or [array cast](#handling-multiple-values-in-a-single-column-as-an-array), you can mutate the state after it has been cast by passing a function to the `castStateUsing()` method: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('price') + ->numeric() + ->castStateUsing(function (float $state): ?float { + if (blank($state)) { + return null; + } + + return round($state * 100); + }) +``` + +You can even access the original state before it was cast, by defining an `$originalState` argument in the function: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('price') + ->numeric() + ->castStateUsing(function (float $state, mixed $originalState): ?float { + // ... + }) +``` + +### Importing relationships + +You may use the `relationship()` method to import a relationship. At the moment, only `BelongsTo` relationships are supported. For example, if you have a `category` column in your CSV, you may want to import the category relationship: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('author') + ->relationship() +``` + +In this example, the `author` column in the CSV will be mapped to the `author_id` column in the database. The CSV should contain the primary keys of authors, usually `id`. + +If the column has a value, but the author cannot be found, the import will fail validation. Filament automatically adds validation to all relationship columns, to ensure that the relationship is not empty when it is required. + +#### Customizing the relationship import resolution + +If you want to find a related record using a different column, you can pass the column name as `resolveUsing`: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('author') + ->relationship(resolveUsing: 'email') +``` + +You can pass in multiple columns to `resolveUsing`, and they will be used to find the author, in an "or" fashion. For example, if you pass in `['email', 'username']`, the record can be found by either their email or username: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('author') + ->relationship(resolveUsing: ['email', 'username']) +``` + +You can also customize the resolution process, by passing in a function to `resolveUsing`, which should return a record to associate with the relationship: + +```php +use App\Models\Author; +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('author') + ->relationship(resolveUsing: function (array $state): ?Author { + return Author::query() + ->where('email', $state) + ->orWhere('username', $state) + ->first(); + }) +``` + +You could even use this function to dynamically determine which columns to use to resolve the record: + +```php +use App\Models\Author; +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('author') + ->relationship(resolveUsing: function (array $state): ?Author { + if (filter_var($state, FILTER_VALIDATE_EMAIL)) { + return 'email'; + } + + return 'username'; + }) +``` + +### Handling multiple values in a single column as an array + +You may use the `array()` method to cast the values in a column to an array. It accepts a delimiter as its first argument, which is used to split the values in the column into an array. For example, if you have a `documentation_urls` column in your CSV, you may want to cast it to an array of URLs: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('documentation_urls') + ->array(',') +``` + +In this example, we pass in a comma as the delimiter, so the values in the column will be split by commas, and cast to an array. + +#### Casting each item in an array + +If you want to cast each item in the array to a different data type, you can chain the [built-in casting methods](#casting-state): + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('customer_ratings') + ->array(',') + ->integer() // Casts each item in the array to an integer. +``` + +#### Validating each item in an array + +If you want to validate each item in the array, you can chain the `nestedRecursiveRules()` method: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('customer_ratings') + ->array(',') + ->integer() + ->rules(['array']) + ->nestedRecursiveRules(['integer', 'min:1', 'max:5']) +``` + +### Customizing how a column is filled into a record + +If you want to customize how column state is filled into a record, you can pass a function to the `fillUsing()` method: + +```php +use App\Models\Product; + +ImportColumn::make('sku') + ->fillUsing(function (Product $record, string $state): void { + $product->state = strtoupper($state); + }) +``` + +## Updating existing records when importing + +When generating an importer class, you will see this `resolveRecord()` method: + +```php +use App\Models\Product; + +public function resolveRecord(): ?Product +{ + // return Product::firstOrNew([ + // // Update existing records, matching them by `$this->data['column_name']` + // 'email' => $this->data['email'], + // ]); + + return new Product(); +} +``` + +This method is called for each row in the CSV, and is responsible for returning a model instance that will be filled with the data from the CSV, and saved to the database. By default, it will create a new record for each row. However, you can customize this behavior to update existing records instead. For example, you might want to update a product if it already exists, and create a new one if it doesn't. To do this, you can uncomment the `firstOrNew()` line, and pass the column name that you want to match on. For a product, we might want to match on the `sku` column: + +```php +use App\Models\Product; + +public function resolveRecord(): ?Product +{ + return Product::firstOrNew([ + 'sku' => $this->data['sku'], + ]); +} +``` + +### Updating existing records when importing only + +If you want to write an importer that only updates existing records, and does not create new ones, you can return `null` if no record is found: + +```php +use App\Models\Product; + +public function resolveRecord(): ?Product +{ + return Product::query() + ->where('sku', $this->data['sku']) + ->first(); +} +``` + +### Ignoring blank state for an import column + +By default, if a column in the CSV is blank, and mapped by the user, and it's not required by validation, the column will be imported as `null` in the database. If you'd like to ignore blank state, and use the existing value in the database instead, you can call the `ignoreBlankState()` method: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('price') + ->ignoreBlankState() +``` + +## Using import options + +The import action can render extra form components that the user can interact with when importing a CSV. This can be useful to allow the user to customize the behavior of the importer. For instance, you might want a user to be able to choose whether to update existing records when importing, or only create new ones. To do this, you can return options form components from the `getOptionsFormComponents()` method on your importer class: + +```php +use Filament\Forms\Components\Checkbox; + +public function getOptionsFormComponents(): array +{ + return [ + Checkbox::make('update_existing') + ->label('Update existing records'), + ]; +} +``` + +Now, you can access the data from these options inside the importer class, by calling `$this->options`. For example, you might want to use it inside `resolveRecord()` to [update an existing product](#updating-existing-records-when-importing): + +```php +use App\Models\Product; + +public function resolveRecord(): ?Product +{ + if ($this->options['update_existing'] ?? false) { + return Product::firstOrNew([ + 'sku' => $this->data['sku'], + ]); + } + + return new Product(); +} +``` + +## Improving import column mapping guesses + +By default, Filament will attempt to "guess" which columns in the CSV match which columns in the database, to save the user time. It does this by attempting to find different combinations of the column name, with spaces, `-`, `_`, all cases insensitively. However, if you'd like to improve the guesses, you can call the `guesses()` method with more examples of the column name that could be present in the CSV: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('sku') + ->guesses(['id', 'number', 'stock-keeping unit']) +``` + +## Providing example CSV data + +Before the user uploads a CSV, they have an option to download example CSV file, containing all the available columns that can be imported. This is useful, as it allows the user import this file directly into their spreadsheet software, and fill it out. + +You can also add an example row to the CSV, to show the user what the data should look like. To fill in this example row, you can pass in an example column value to the `example()` method: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('sku') + ->example('ABC123') +``` + +## Limiting the maximum number of rows that can be imported + +To prevent server overload, you may wish to limit the maximum number of rows that can be imported from one CSV file. You can do this by calling the `maxRows()` method on the action: + +```php +ImportAction::make() + ->importer(ProductImporter::class) + ->maxRows(100000) +``` + +## Changing the import chunk size + +Filament will chunk the CSV, and process each chunk in a different queued job. By default, chunks are 100 rows at a time. You can change this by calling the `chunkSize()` method on the action: + +```php +ImportAction::make() + ->importer(ProductImporter::class) + ->chunkSize(250) +``` + +If you are encountering memory issues when importing large CSV files, you may wish to reduce the chunk size. + +## Customizing the import job + +The default job for processing imports is `Filament\Actions\Imports\Jobs\ImportCsv`. If you want to extend this class and override any of its methods, you may replace the original class in the `register()` method of a service provider: + +```php +use App\Jobs\ImportCsv; +use Filament\Actions\Imports\Jobs\ImportCsv::class as BaseImportCsv; + +$this->app->bind(BaseImportCsv::class, ImportCsv::class); +``` + +Or, you can pass the new job class to the `job()` method on the action, to customize the job for a specific import: + +```php +use App\Jobs\ImportCsv; + +ImportAction::make() + ->importer(ProductImporter::class) + ->job(ImportCsv::class) +``` + +### Customizing the import job middleware + +By default, the import system will only process one job at a time from each import. This is to prevent the server from being overloaded, and other jobs from being delayed by large imports. That functionality is defined in the `WithoutOverlapping` middleware on the importer class: + +```php +public function getJobMiddleware(): array +{ + return [ + (new WithoutOverlapping("import{$this->import->id}"))->expireAfter(600), + ]; +} +``` + +If you'd like to customize the middleware that is applied to jobs of a certain importer, you may override this method in your importer class. You can read more about job middleware in the [Laravel docs](https://laravel.com/docs/queues#job-middleware). + +### Customizing the import job retries + +By default, the import system will retry a job for 24 hours. This is to allow for temporary issues, such as the database being unavailable, to be resolved. That functionality is defined in the `getJobRetryUntil()` method on the importer class: + +```php +use Carbon\CarbonInterface; + +public function getJobRetryUntil(): CarbonInterface +{ + return now()->addDay(); +} +``` + +If you'd like to customize the retry time for jobs of a certain importer, you may override this method in your importer class. You can read more about job retries in the [Laravel docs](https://laravel.com/docs/queues#time-based-attempts). + +### Customizing the import job tags + +By default, the import system will tag each job with the ID of the import. This is to allow you to easily find all jobs related to a certain import. That functionality is defined in the `getJobTags()` method on the importer class: + +```php +public function getJobTags(): array +{ + return ["import{$this->import->id}"]; +} +``` + +If you'd like to customize the tags that are applied to jobs of a certain importer, you may override this method in your importer class. + +## Customizing import validation messages + +The import system will automatically validate the CSV file before it is imported. If there are any errors, the user will be shown a list of them, and the import will not be processed. If you'd like to override any default validation messages, you may do so by overriding the `getValidationMessages()` method on your importer class: + +```php +public function getValidationMessages(): array +{ + return [ + 'name.required' => 'The name column must not be empty.', + ]; +} +``` + +To learn more about customizing validation messages, read the [Laravel docs](https://laravel.com/docs/validation#customizing-the-error-messages). + +### Customizing import validation attributes + +When columns fail validation, their label is used in the error message. To customize the label used in field error messages, use the `validationAttribute()` method: + +```php +use Filament\Actions\Imports\ImportColumn; + +ImportColumn::make('name') + ->validationAttribute('full name') +``` + +## Lifecycle hooks + +Hooks may be used to execute code at various points within an importer's lifecycle, like before a record is saved. To set up a hook, create a protected method on the importer class with the name of the hook: + +```php +protected function beforeSave(): void +{ + // ... +} +``` + +In this example, the code in the `beforeSave()` method will be called before the validated data from the CSV is saved to the database. + +There are several available hooks for importers: + +```php +use Filament\Actions\Imports\Importer; + +class ProductImporter extends Importer +{ + // ... + + protected function beforeValidate(): void + { + // Runs before the CSV data for a row is validated. + } + + protected function afterValidate(): void + { + // Runs after the CSV data for a row is validated. + } + + protected function beforeFill(): void + { + // Runs before the validated CSV data for a row is filled into a model instance. + } + + protected function afterFill(): void + { + // Runs after the validated CSV data for a row is filled into a model instance. + } + + protected function beforeSave(): void + { + // Runs before a record is saved to the database. + } + + protected function beforeCreate(): void + { + // Similar to `beforeSave()`, but only runs when creating a new record. + } + + protected function beforeUpdate(): void + { + // Similar to `beforeSave()`, but only runs when updating an existing record. + } + + protected function afterSave(): void + { + // Runs after a record is saved to the database. + } + + protected function afterCreate(): void + { + // Similar to `afterSave()`, but only runs when creating a new record. + } + + protected function afterUpdate(): void + { + // Similar to `afterSave()`, but only runs when updating an existing record. + } +} +``` + +Inside these hooks, you can access the current row's data using `$this->data`. You can also access the original row of data from the CSV, before it was [cast](#casting-state) or mapped, using `$this->originalData`. + +The current record (if it exists yet) is accessible in `$this->record`, and the [options](#using-import-options) using `$this->options`. diff --git a/packages/actions/src/ImportAction.php b/packages/actions/src/ImportAction.php index 447e15d762..b8be6ffa59 100644 --- a/packages/actions/src/ImportAction.php +++ b/packages/actions/src/ImportAction.php @@ -335,12 +335,12 @@ class ImportAction extends Action public function chunkSize(int | Closure $size): static { - $this->size = $size; + $this->chunkSize = $size; return $this; } - public function max(int | Closure | null $rows): static + public function maxRows(int | Closure | null $rows): static { $this->maxRows = $rows; diff --git a/packages/actions/src/Imports/ImportColumn.php b/packages/actions/src/Imports/ImportColumn.php index bbb56b5208..ce60a86ef3 100644 --- a/packages/actions/src/Imports/ImportColumn.php +++ b/packages/actions/src/Imports/ImportColumn.php @@ -36,7 +36,7 @@ class ImportColumn extends Component protected ?Closure $fillRecordUsing = null; - protected ?Closure $sanitizeStateUsing = null; + protected ?Closure $castStateUsing = null; /** * @var array | Closure @@ -64,6 +64,8 @@ class ImportColumn extends Component */ protected array $resolvedRelatedRecords = []; + protected string | Closure | null $validationAttribute = null; + final public function __construct(string $name) { $this->name($name); @@ -80,9 +82,9 @@ class ImportColumn extends Component public function getSelect(): Select { return Select::make($this->getName()) - ->label($this->label) + ->label($this->getLabel()) ->placeholder(__('filament-actions::import.modal.form.columns.placeholder')) - ->required($this->isMappingRequired); + ->required($this->isMappingRequired()); } public function name(string $name): static @@ -121,6 +123,13 @@ class ImportColumn extends Component return $this; } + public function integer(bool | Closure $condition = true): static + { + $this->numeric($condition, decimalPlaces: 0); + + return $this; + } + public function boolean(bool | Closure $condition = true): static { $this->isBoolean = $condition; @@ -203,9 +212,9 @@ class ImportColumn extends Component }, []); } - public function sanitizeStateUsing(?Closure $callback): static + public function castStateUsing(?Closure $callback): static { - $this->sanitizeStateUsing = $callback; + $this->castStateUsing = $callback; return $this; } @@ -220,21 +229,21 @@ class ImportColumn extends Component /** * @param array $options */ - public function sanitizeState(mixed $state, array $options): mixed + public function castState(mixed $state, array $options): mixed { $originalState = $state; if (filled($arraySeparator = $this->getArraySeparator())) { $state = collect(explode($arraySeparator, strval($state))) - ->map(fn (mixed $stateItem): mixed => $this->sanitizeStateItem($stateItem)) + ->map(fn (mixed $stateItem): mixed => $this->castStateItem($stateItem)) ->filter(fn (mixed $stateItem): bool => filled($stateItem)) ->all(); } else { - $state = $this->sanitizeStateItem($state); + $state = $this->castStateItem($state); } - if ($this->sanitizeStateUsing) { - return $this->evaluate($this->sanitizeStateUsing, [ + if ($this->castStateUsing) { + return $this->evaluate($this->castStateUsing, [ 'originalState' => $originalState, 'state' => $state, 'options' => $options, @@ -316,6 +325,10 @@ class ImportColumn extends Component 'state' => $state, ]); + if (blank($resolveUsing)) { + return $this->resolvedRelatedRecords[$state] = null; + } + if ($resolveUsing instanceof Model) { return $this->resolvedRelatedRecords[$state] = $resolveUsing; } @@ -422,12 +435,34 @@ class ImportColumn extends Component return $this->getImporter()->getRecord(); } + public function isMappingRequired(): bool + { + return (bool) $this->evaluate($this->isMappingRequired); + } + public function hasRelationship(): bool { return filled($this->getRelationshipName()); } - protected function sanitizeStateItem(mixed $state): mixed + public function validationAttribute(string | Closure | null $label): static + { + $this->validationAttribute = $label; + + return $this; + } + + public function getValidationAttribute(): string + { + return $this->evaluate($this->validationAttribute) ?? Str::lcfirst($this->getLabel()); + } + + public function getLabel(): ?string + { + return $this->evaluate($this->label); + } + + protected function castStateItem(mixed $state): mixed { if (is_string($state)) { $state = trim($state); @@ -438,17 +473,17 @@ class ImportColumn extends Component } if ($this->isBoolean()) { - return $this->sanitizeBooleanStateItem($state); + return $this->castBooleanStateItem($state); } if ($this->isNumeric()) { - return $this->sanitizeNumericStateItem($state); + return $this->castNumericStateItem($state); } return $state; } - protected function sanitizeBooleanStateItem(mixed $state): bool + protected function castBooleanStateItem(mixed $state): bool { // Narrow down the possible values of the state to make comparison easier. $state = strtolower(strval($state)); @@ -460,7 +495,7 @@ class ImportColumn extends Component }; } - protected function sanitizeNumericStateItem(mixed $state): int | float + protected function castNumericStateItem(mixed $state): int | float { $state = floatval(preg_replace('/[^0-9.]/', '', $state)); diff --git a/packages/actions/src/Imports/Importer.php b/packages/actions/src/Imports/Importer.php index 6476ee0c9b..24efae0098 100644 --- a/packages/actions/src/Imports/Importer.php +++ b/packages/actions/src/Imports/Importer.php @@ -169,7 +169,19 @@ abstract class Importer */ public function getValidationAttributes(): array { - return []; + $attributes = []; + + foreach ($this->getCachedColumns() as $column) { + $validationAttribute = $column->getValidationAttribute(); + + if (blank($validationAttribute)) { + continue; + } + + $attributes[$column->getName()] = $validationAttribute; + } + + return $attributes; } public function fillRecord(): void diff --git a/packages/forms/docs/05-validation.md b/packages/forms/docs/05-validation.md index 4ab7c5b93a..4233b35269 100644 --- a/packages/forms/docs/05-validation.md +++ b/packages/forms/docs/05-validation.md @@ -496,7 +496,7 @@ TextInput::make('slug')->rules([ ]) ``` -## Validation attributes +## Customizing validation attributes When fields fail validation, their label is used in the error message. To customize the label used in field error messages, use the `validationAttribute()` method: diff --git a/packages/notifications/docs/05-customizing-notifications.md b/packages/notifications/docs/05-customizing-notifications.md index 630afabc09..f66df0e0b4 100644 --- a/packages/notifications/docs/05-customizing-notifications.md +++ b/packages/notifications/docs/05-customizing-notifications.md @@ -101,7 +101,7 @@ class Notification extends BaseNotification } ``` -Next, you should bind your custom `Notification` class into the container inside a service provider's `boot()` method: +Next, you should bind your custom `Notification` class into the container inside a service provider's `register()` method: ```php use App\Notifications\Notification;