RemoveDuplicates.org

Remove duplicate lines from a text list, or duplicate rows from CSV/TSV text such as rows copied from Excel or Google Sheets. Uses the same engine as removeduplicates.org and returns the cleaned text with counts. Stateless: the text is processed in memory for this call only and is never stored.

{
  "type": "object",
  "properties": {
    "text": {
      "type": "string",
      "description": "The list or table text: one item per line, or CSV/tab-separated rows. Up to 131072 UTF-8 bytes."
    },
    "format": {
      "type": "string",
      "enum": [
        "auto",
        "lines",
        "table"
      ],
      "default": "auto",
      "description": "auto detects CSV/TSV tables; lines compares whole lines; table forces CSV/TSV parsing."
    },
    "ignoreCase": {
      "type": "boolean",
      "default": false,
      "description": "Treat values such as Apple and apple as duplicates."
    },
    "trim": {
      "type": "boolean",
      "default": false,
      "description": "Ignore surrounding spaces when comparing and return trimmed values."
    },
    "removeEmpty": {
      "type": "boolean",
      "default": true,
      "description": "Drop blank lines or all-blank rows."
    },
    "keep": {
      "type": "string",
      "enum": [
        "first",
        "last"
      ],
      "default": "first",
      "description": "Which occurrence of a duplicate survives."
    },
    "order": {
      "type": "string",
      "enum": [
        "preserve",
        "sort"
      ],
      "default": "preserve",
      "description": "preserve keeps the original order; sort returns a natural (numeric-aware) sort."
    },
    "compare": {
      "anyOf": [
        {
          "type": "string",
          "enum": [
            "row"
          ]
        },
        {
          "type": "integer",
          "minimum": 0
        },
        {
          "type": "array",
          "items": {
            "type": "integer",
            "minimum": 0
          },
          "minItems": 1,
          "uniqueItems": true
        }
      ],
      "default": "row",
      "description": "Tables only: \"row\" compares entire rows; a zero-based column index or an array of indices compares only those columns."
    },
    "header": {
      "type": "boolean",
      "default": false,
      "description": "Tables only: keep the first row as a header outside duplicate checks and counts."
    }
  },
  "required": [
    "text"
  ],
  "additionalProperties": false
}

{
  "type": "object",
  "properties": {
    "text": {
      "type": "string",
      "description": "The cleaned text, rows joined with \\n."
    },
    "format": {
      "type": "object",
      "properties": {
        "kind": {
          "type": "string",
          "enum": [
            "lines",
            "csv",
            "tsv"
          ]
        },
        "columns": {
          "type": "integer"
        },
        "header": {
          "type": "boolean"
        }
      },
      "required": [
        "kind",
        "columns",
        "header"
      ]
    },
    "stats": {
      "type": "object",
      "properties": {
        "total": {
          "type": "integer",
          "description": "Input lines or data rows (header excluded)."
        },
        "unique": {
          "type": "integer",
          "description": "Lines or data rows kept."
        },
        "removed": {
          "type": "integer",
          "description": "duplicates + emptyRemoved."
        },
        "duplicates": {
          "type": "integer"
        },
        "emptyRemoved": {
          "type": "integer"
        },
        "reduction": {
          "type": "number",
          "description": "Percent of input removed, one decimal."
        }
      },
      "required": [
        "total",
        "unique",
        "removed",
        "duplicates",
        "emptyRemoved",
        "reduction"
      ]
    }
  },
  "required": [
    "text",
    "format",
    "stats"
  ]
}