From ef3268c938271746253a70d8e182a6c9aa957d9c Mon Sep 17 00:00:00 2001 From: Nico Wagner Date: Mon, 3 Jul 2023 09:08:49 +0200 Subject: [PATCH] Remove `json` command --- CHANGELOG.md | 1 + README.md | 36 ----- docs/book/src/de/SUMMARY.md | 1 - src/bin/pica/commands/json.rs | 91 ------------ src/bin/pica/commands/mod.rs | 2 - src/bin/pica/config.rs | 1 - src/bin/pica/main.rs | 6 +- tests/pica/json.rs | 251 ---------------------------------- tests/pica/mod.rs | 1 - 9 files changed, 2 insertions(+), 388 deletions(-) delete mode 100644 src/bin/pica/commands/json.rs delete mode 100644 tests/pica/json.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1591e2c32..a339592bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed * #639 Remove `xml` command +* #640 Remove `json` command ## [0.17.0] - 2023-06-30 diff --git a/README.md b/README.md index 5db0cc1bc..221c6a54d 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,6 @@ $ cargo install --git https://github.com/deutsche-nationalbibliothek/pica-rs --t | [select](#select) | beta | select subfield values from records | | [slice](#slice) | stable | return records withing a range (half-open interval) | | [split](#split) | stable | split a list of records into chunks | -| [json](#json) | depricated | serialize records in JSON | ## Usage @@ -345,41 +344,6 @@ outdir ├── ... ``` -### JSON - -This command serializes the internal representation of record to JSON: - -```bash -$ echo -e "003@ \x1f0123456789\x1fab\x1e" | pica json | jq . -[ - { - "fields": [ - { - "name": "003@", - "subfields": [ - { - "name": "0", - "value": "123456789" - }, - { - "name": "a", - "value": "b" - } - ] - } - ] - } -] -``` - -The result can be processed with other tools and programming languages. -To get [PICA JSON](http://format.gbv.de/pica/json) format you can pipe -the result to this [jq](https://stedolan.github.io/jq/) command: - -```bash -jq -c '.[]|.fields|map([.tag,.occurrence]+(.subfields|map(.tag,.value)))' -``` - ## Related Projects - [Catmandu::Pica](https://metacpan.org/pod/Catmandu::PICA) - Catmandu modules for working with PICA+ data diff --git a/docs/book/src/de/SUMMARY.md b/docs/book/src/de/SUMMARY.md index f0180732b..a49519502 100644 --- a/docs/book/src/de/SUMMARY.md +++ b/docs/book/src/de/SUMMARY.md @@ -18,7 +18,6 @@ - [select]() - [slice]() - [split]() - - [json]() # Tutorials diff --git a/src/bin/pica/commands/json.rs b/src/bin/pica/commands/json.rs deleted file mode 100644 index d4f262570..000000000 --- a/src/bin/pica/commands/json.rs +++ /dev/null @@ -1,91 +0,0 @@ -use std::ffi::OsString; -use std::io::{self, Read, Write}; - -use clap::Parser; -use pica::{PicaWriter, Reader, ReaderBuilder, WriterBuilder}; -use serde::{Deserialize, Serialize}; - -use crate::config::Config; -use crate::skip_invalid_flag; -use crate::translit::translit_maybe; -use crate::util::CliResult; - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) struct JsonConfig { - pub(crate) skip_invalid: Option, -} - -#[derive(Parser, Debug)] -pub(crate) struct Json { - /// Skip invalid records that can't be decoded - #[arg(short, long)] - skip_invalid: bool, - - /// Transliterate output into the selected normalform - /// (possible values: "nfd", "nfkd", "nfc" and "nfkc") - #[arg(long, - value_name = "NF", - value_parser = ["nfd", "nfkd", "nfc", "nfkc"], - hide_possible_values = true, - )] - translit: Option, - - /// Write output to instead of stdout - #[arg(short, long, value_name = "filename")] - output: Option, - - /// Read one or more files in normalized PICA+ format. - #[arg(default_value = "-", hide_default_value = true)] - filenames: Vec, -} - -impl Json { - pub(crate) fn run(self, config: &Config) -> CliResult<()> { - eprintln!( - "WARNING: The `json` command will be removed in version 0.17, \ - please use the `convert` command instead." - ); - - let skip_invalid = skip_invalid_flag!( - self.skip_invalid, - config.json, - config.global - ); - - let mut writer: Box = - WriterBuilder::new().from_path_or_stdout(self.output)?; - writer.write_all(b"[")?; - - for filename in self.filenames { - let builder = - ReaderBuilder::new().skip_invalid(skip_invalid); - let mut reader: Reader> = match filename - .to_str() - { - Some("-") => builder.from_reader(Box::new(io::stdin())), - _ => builder.from_path(filename)?, - }; - - for (count, result) in reader.records().enumerate() { - let record = result?; - - if count > 0 { - writer.write_all(b",")?; - } - - let j = translit_maybe( - &serde_json::to_string(&record).unwrap(), - self.translit.as_deref(), - ); - - writer.write_all(j.as_bytes())?; - } - } - - writer.write_all(b"]")?; - writer.flush()?; - - Ok(()) - } -} diff --git a/src/bin/pica/commands/mod.rs b/src/bin/pica/commands/mod.rs index f89666210..6d2e9b007 100644 --- a/src/bin/pica/commands/mod.rs +++ b/src/bin/pica/commands/mod.rs @@ -6,7 +6,6 @@ mod filter; mod frequency; mod hash; mod invalid; -mod json; mod partition; mod print; mod sample; @@ -22,7 +21,6 @@ pub(crate) use filter::{Filter, FilterConfig}; pub(crate) use frequency::{Frequency, FrequencyConfig}; pub(crate) use hash::{Hash, HashConfig}; pub(crate) use invalid::Invalid; -pub(crate) use json::{Json, JsonConfig}; pub(crate) use partition::{Partition, PartitionConfig}; pub(crate) use print::{Print, PrintConfig}; pub(crate) use sample::{Sample, SampleConfig}; diff --git a/src/bin/pica/config.rs b/src/bin/pica/config.rs index 0afe0812e..3d907cf42 100644 --- a/src/bin/pica/config.rs +++ b/src/bin/pica/config.rs @@ -37,7 +37,6 @@ pub(crate) struct Config { pub(crate) filter: Option, pub(crate) frequency: Option, pub(crate) hash: Option, - pub(crate) json: Option, pub(crate) partition: Option, pub(crate) print: Option, pub(crate) sample: Option, diff --git a/src/bin/pica/main.rs b/src/bin/pica/main.rs index 72c972ba2..38d87d77b 100644 --- a/src/bin/pica/main.rs +++ b/src/bin/pica/main.rs @@ -18,7 +18,7 @@ use std::{io, process}; use clap::{CommandFactory, Parser, Subcommand}; use commands::{ Cat, Completions, Convert, Count, Filter, Frequency, Hash, Invalid, - Json, Partition, Print, Sample, Select, Slice, Split, + Partition, Print, Sample, Select, Slice, Split, }; use config::Config; use util::{CliError, CliResult}; @@ -49,9 +49,6 @@ enum Commands { Frequency(Frequency), Hash(Hash), Invalid(Invalid), - - /// Serialize records to JSON - Json(Json), Partition(Partition), Print(Print), @@ -77,7 +74,6 @@ fn run() -> CliResult<()> { Commands::Frequency(cmd) => cmd.run(&config), Commands::Hash(cmd) => cmd.run(&config), Commands::Invalid(cmd) => cmd.run(&config), - Commands::Json(cmd) => cmd.run(&config), Commands::Partition(cmd) => cmd.run(&config), Commands::Print(cmd) => cmd.run(&config), Commands::Sample(cmd) => cmd.run(&config), diff --git a/tests/pica/json.rs b/tests/pica/json.rs deleted file mode 100644 index d78790fd6..000000000 --- a/tests/pica/json.rs +++ /dev/null @@ -1,251 +0,0 @@ -use std::fs::read_to_string; -use std::path::Path; - -use assert_cmd::Command; -use predicates::prelude::*; -use tempfile::Builder; - -use crate::common::{CommandExt, TestContext, TestResult}; - -const DEPRICATION_WARNING: &str = "WARNING: The `json` command will be removed in version 0.17, please use the `convert` command instead."; - -#[test] -fn pica_json_single_record() -> TestResult { - let mut cmd = Command::cargo_bin("pica")?; - let assert = - cmd.arg("json").arg("tests/data/1004916019.dat").assert(); - - let expected = - read_to_string("tests/data/1004916019.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - Ok(()) -} - -#[test] -fn pica_json_multiple_records() -> TestResult { - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("--skip-invalid") - .arg("tests/data/dump.dat.gz") - .assert(); - - let expected = read_to_string("tests/data/dump.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("tests/data/1004916019.dat") - .arg("tests/data/000008672.dat") - .assert(); - - let expected = - predicate::path::eq_file(Path::new("tests/data/tworecs.json")); - assert - .success() - .stderr(predicate::str::starts_with(DEPRICATION_WARNING)) - // .stderr(predicate::str::is_empty()) - .stdout(expected); - - let data = read_to_string("tests/data/1004916019.dat").unwrap(); - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("-") - .arg("tests/data/000008672.dat") - .write_stdin(data) - .assert(); - - let expected = - predicate::path::eq_file(Path::new("tests/data/tworecs.json")); - assert - .success() - .stderr(predicate::str::starts_with(DEPRICATION_WARNING)) - .stdout(expected); - - Ok(()) -} - -#[test] -fn pica_json_stdin() -> TestResult { - let data = read_to_string("tests/data/1004916019.dat").unwrap(); - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd.arg("json").arg("-").write_stdin(data).assert(); - - let expected = predicate::path::eq_file(Path::new( - "tests/data/1004916019.json", - )); - - assert - .success() - .stderr(predicate::str::starts_with(DEPRICATION_WARNING)) - .stdout(expected); - - let data = read_to_string("tests/data/1004916019.dat").unwrap(); - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd.arg("json").write_stdin(data).assert(); - - let expected = predicate::path::eq_file(Path::new( - "tests/data/1004916019.json", - )); - - assert - .success() - .stderr(predicate::str::starts_with(DEPRICATION_WARNING)) - .stdout(expected); - - Ok(()) -} - -#[test] -fn pica_json_write_output() -> TestResult { - let filename = Builder::new().suffix(".json").tempfile()?; - let filename_str = filename.path(); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("--output") - .arg(filename_str) - .arg("tests/data/1004916019.dat") - .assert(); - assert.success(); - - let expected = - read_to_string("tests/data/1004916019.json").unwrap(); - let actual = read_to_string(filename_str).unwrap(); - assert_eq!(expected.trim_end().to_string(), actual); - - Ok(()) -} - -#[test] -fn pica_json_translit() -> TestResult { - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("tests/data/004732650-reduced.dat.gz") - .assert(); - - let expected = - read_to_string("tests/data/004732650-nfd.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - let expected = vec![ - ("nfd", "tests/data/004732650-nfd.json"), - ("nfkd", "tests/data/004732650-nfd.json"), - ("nfc", "tests/data/004732650-nfc.json"), - ("nfkc", "tests/data/004732650-nfc.json"), - ]; - - for (translit, output) in expected { - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("--translit") - .arg(translit) - .arg("tests/data/004732650-reduced.dat.gz") - .assert(); - - let expected = read_to_string(output).unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - } - - Ok(()) -} - -#[test] -fn pica_json_skip_invalid() -> TestResult { - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .arg("json") - .arg("--skip-invalid") - .arg("tests/data/invalid.dat") - .assert(); - assert.success().stdout(predicate::eq("[]")); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd.arg("json").arg("tests/data/dump.dat.gz").assert(); - assert - .failure() - .code(1) - .stderr(predicate::str::starts_with(DEPRICATION_WARNING)) - .stderr(predicate::str::contains( - "Pica Error: Invalid record on line 2.\n", - )); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .with_config( - &TestContext::new(), - r#"[json] -skip-invalid = true -"#, - ) - .arg("json") - .arg("tests/data/1004916019.dat") - .assert(); - - let expected = - read_to_string("tests/data/1004916019.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .with_config( - &TestContext::new(), - r#"[global] -skip-invalid = true -"#, - ) - .arg("json") - .arg("tests/data/1004916019.dat") - .assert(); - - let expected = - read_to_string("tests/data/1004916019.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .with_config( - &TestContext::new(), - r#"[global] -skip-invalid = false - -[json] -skip-invalid = true -"#, - ) - .arg("json") - .arg("tests/data/1004916019.dat") - .assert(); - - let expected = - read_to_string("tests/data/1004916019.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - let mut cmd = Command::cargo_bin("pica")?; - let assert = cmd - .with_config( - &TestContext::new(), - r#"[global] -skip-invalid = false - -[json] -skip-invalid = false -"#, - ) - .arg("json") - .arg("--skip-invalid") - .arg("tests/data/1004916019.dat") - .assert(); - - let expected = - read_to_string("tests/data/1004916019.json").unwrap(); - assert.success().stdout(expected.trim_end().to_string()); - - Ok(()) -} diff --git a/tests/pica/mod.rs b/tests/pica/mod.rs index 089ce82c6..684fc38ef 100644 --- a/tests/pica/mod.rs +++ b/tests/pica/mod.rs @@ -3,7 +3,6 @@ use predicates::prelude::*; use crate::TestResult; -mod json; mod sample; #[test]