Importer

CSV

    use Catmandu::Importer::CSV;

    my $importer = Catmandu::Importer::CSV->new(file => "/foo/bar.csv");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

Convert CSV to other formats with the catmandu command line client:

    # convert CSV file to JSON
    catmandu convert CSV to JSON < journals.csv
    # set column names if CSV file has no header line
    echo '12157,"The Journal of Headache and Pain",2193-1801' | \
      catmandu convert CSV --header 0 --fields 'id,title,issn' to YAML
    # set field separator and quote character 
    echo '12157;$The Journal of Headache and Pain$;2193-1801' | \
      catmandu convert CSV --header 0 --fields 'id,title,issn' --sep_char ';' --quote_char '$' to XLSX --file journal.xlsx

JSON

    use Catmandu::Importer::JSON;

    my $importer = Catmandu::Importer::JSON->new(file => "/foo/bar.json");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

The defaults assume a newline delimited JSON file:

    { "recordno": 1, "name": "Alpha" }
    { "recordno": 2, "name": "Beta" }
    { "recordno": 3, "name": "Gamma" }

Use the multiline or array options to parse pretty-printed JSON or JSON arrays.

Mock

    use Catmandu::Importer::Mock;

    my $importer = Catmandu::Importer::Mock->new();

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

Modules

Multi

    use Catmandu::Importer::Multi;

    my $importer = Catmandu::Importer::Multi->new(importers => [
        Catmandu::Importer::Mock->new,
        Catmandu::Importer::Mock->new,
    ]);

    my $importer = Catmandu::Importer::Multi->new(
        'importer1',
        'importer2',
    );

    # return all the items of each importer in turn
    $importer->each(sub {
        # ...
    });

Null

    # On the command line
    catmandu convert Null --fix 'add_field(foo,bar)'   
    # creates { "foo": "bar" }

    # In perl
    use Catmandu::Importer::Null;

    my $importer = Catmandu::Importer::Null->new();

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

Text

With catmandu command line client:

    # separate fields by whitespace sequences just like awk
    catmandu convert Text --split '\s+' 

    # import all lines starting with '#', omitting this character 
    catmandu convert Text --pattern '^#(.*)'

In Perl code:

    use Catmandu::Importer::Text;

    my $importer = Catmandu::Importer::text->new( file => "/foo/bar.txt" );

    # print all lines with line number
    $importer->each(sub {
        my $item = $_[0];
        printf "%d: %s" , $item->{_id} , $item->{text};  
    });

YAML

    use Catmandu::Importer::YAML;

    my $importer = Catmandu::Importer::YAML->new(file => "/foo/bar.yaml");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

    The YAML input file needs to be separated into records:

    ---
    - recordno: 1
    - name: Alpha
    ---
    - recordno: 2
    - name: Beta
    ...

    where '---' is the record separator and '...' the EOF indicator.

AlephX

    use Catmandu::Importer::AlephX;

    my $importer = Catmandu::Importer::AlephX->new(
                        url => 'http://ram19:8995/X' ,
                        query => 'WRD=(art)' ,
                        base => 'usm01' ,
                        );

    my $n = $importer->each(sub {
        my $r = $_[0];
        # ...
        say Dumper($r->{record});
        say Dumper($r->{items});
    });

ArXiv

  use Catmandu::Importer::ArXiv;

  my %attrs = (
    query => 'all:electron'
  );

  my $importer = Catmandu::Importer::ArXiv->new(%attrs);

  my $n = $importer->each(sub {
    my $hashref = $_[0];
    # ...
  });

Atom

    use Catmandu::Importer::Atom;

    my $importer = Catmandu::Importer::Atom->new(url => "...");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

BagIt

   use Catmandu::Importer::BagIt

   my $importer = Catmandu::Importer::BagIt->new(
                        bags => "/my/bags/*" ,
                  );

   my $importer = Catmandu::Importer::BagIt->new(
                        bags => ["directory1","directory2"] ,
                        include_manifests => 0 ,
                        include_payloads  => 0 ,
                        verify            => 1
                  );

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

To convert BagIt directories into a JSON representation with the catmandu command line client:

    # Use a glob to find all directories in /my/path/
    catmandu convert BagIt --bags '/my/path/*' --verify 1 

BibTeX

    use Catmandu::Importer::BibTeX;

    my $importer = Catmandu::Importer::BibTeX->new(file => "/foo/bar.bib");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    }); 

Blacklight

    # From the command line
    $ catmandu convert Blacklight --url http://lib.ugent.be/catalog -q Schopenhauer

    # In perl
    use Catmandu::Importer::Blacklight;

    my $importer = Catmandu::Importer::Blacklight->new(
                    url => "...",
                    q   => "..."
      );

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

CrossRef

  use Catmandu::Importer::CrossRef;

  my %attrs = (
    doi => '<doi>',
    usr => '<your-crossref-username>',
    pwd => '<your-crossref-password>',
    fmt => '<xsd_xml | unixref | unixsd | info>'
  );

  my $importer = Catmandu::Importer::CrossRef->new(%attrs);

  my $n = $importer->each(sub {
    my $hashref = $_[0];
    # do something here
  });

DBI

 use Catmandu::Importer::DBI;

 my %attrs = (
        dsn => 'dbi:mysql:foobar' ,
        user => 'foo' ,
        password => 'bar' ,
        query => 'select * from table'
 );

 my $importer = Catmandu::Importer::DBI->new(%attrs);

 # Optional set extra parameters on the database handle
 # $importer->dbh->{LongReadLen} = 1024 * 64;

 $importer->each(sub {
        my $row_hash = shift;
        ...
 });


 # or

 $ catmandu convert DBI --dsn dbi:mysql:foobar --user foo --password bar --query "select * from table"

EuropePMC

  use Catmandu::Importer::EuropePMC;

  my %attrs = (
    source => 'MED',
    query => 'malaria',
    module => 'search',
    db => 'EMBL',
    page => '2',
  );

  my $importer = Catmandu::Importer::EuropePMC->new(%attrs);

  my $n = $importer->each(sub {
    my $hashref = $_[0];
    # ...
  });

ApacheLog

    #!/usr/bin/env perl
    use Catmandu::Importer::ApacheLog;
    use Data::Dumper;

    my $importer = Catmandu::Importer::ApacheLog->new(
        file => "/var/log/httpd/access_log"
    );

    $importer->each(sub{
        print Dumper(shift);
    });

    #!/bin/bash
    catmandu convert ApacheLog --file access.log to YAML

CPAN

 use Catmandu::Importer::CPAN;
 my $importer = Catmandu::Importer::CPAN->new( prefix => 'Catmandu' );

 $importer->each(sub {
    my $module = shift;
    print $module->{distribution} , "\n";
    print $module->{version} , "\n";
    print $module->{date} , "\n";
 });

Or with the catmandu command line client:

  $ catmandu convert CPAN --author NICS --fields distribution,date to CSV

MWTemplates

Command line client catmandu:

catmandu convert MWTemplates --file example.wiki 
catmandu convert MWTemplates --site en --page Feminis
catmandu convert MWTemplates --site de --page Feminism --template Literatur

Parltrack

    catmandu convert Parltrack --dossier "2011/0167(NLE)"
    catmandu convert Parltrack --mep "Rebecca HARMS"
    catmandu convert Parltrack --meps 1

    echo '/dossier/2011/0167(NLE)' | catmandu convert Parltrack
    echo '/mep/Rebecca HARMS' | catmandu convert Parltrack

Purr

 use Catmandu::Importer::Purr;

 my $importer = Catmandu::Importer::Purr->new(prefix => 'Catmandu');
 
 $importer->each(sub {
        my $module = shift;
        print $module->{id} , "\n";
        print $module->{date} , "\n";
        print $module->{distribution} , "\n";
        print $module->{version} , "\n";
        print $module->{abstract} , "\n";
 });
 
 # or

 $ catmandu convert Purr

getJSON

The following three examples are equivalent:

    Catmandu::Importer::getJSON->new(
        file => \"http://example.org/alice.json\nhttp://example.org/bob.json"
    )->each(sub { my ($record) = @_; ... );

    Catmandu::Importer::getJSON->new(
        url  => "http://example.org",
        file => \"/alice.json\n/bob.json"
    )->each(sub { my ($record) = @_; ... );
    
    Catmandu::Importer::getJSON->new(
        url  => "http://example.org/{name}.json",
        file => \"{\"name\":\"alice\"}\n{\"name\":\"bob\"}"
    )->each(sub { my ($record) = @_; ... );

For more convenience the catmandu command line client can be used:

    echo http://example.org/alice.json | catmandu convert getJSON to YAML
    catmandu convert getJSON --from http://example.org/alice.json to YAML
    catmandu convert getJSON --dry 1 --url http://{domain}/robots.txt < domains

Inspire

  use Catmandu::Importer::Inspire;

  my %attrs = (
    id => '1203476',
    fmt => 'endnote',
  );

  my $importer = Catmandu::Importer::Inspire->new(%attrs);

  my $n = $importer->each(sub {
    my $hashref = $_[0];
    # ...
  });

LDAP

    use Catmandu::Importer::LDAP;

    my $importer = Catmandu::Importer::LDAP->new(
        base => "...",
        password => "...",
        search_base => "...",
        search_filter => "(&(...)(...))",
        attributes => {
            name => 1,
            # or
            name => {as => "Name"},
            # or
            name => {as => "Name", array => 1},
        },
    );

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

MAB2

    use Catmandu::Importer::MAB2;

    my $importer = Catmandu::Importer::MAB2->new(file => "./t/mab2.dat", type=> "raw");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

To convert between MAB2 syntax variants with the catmandu command line client:

    catmandu convert MAB2 --type raw to MAB2 --type xml < mab2.dat

MARC

    # From the command line
    $ catmandu convert MARC --fix "marc_map('245a','title')" < /foo/bar.mrc

    # From Perl
    use Catmandu;

    # import records from file
    my $importer = Catmandu->importer('MARC',file => '/foo/bar.mrc');
    my $fixer    = Catmandu->fixer("marc_map('245a','title')");

    $importer->each(sub {
        my $item = shift;
        ...
    });

    # or using the fixer

    $fixer->fix($importer)->each(sub {
        my $item = shift;
        printf "title: %s\n" , $item->{title};
    });

MODS

  use Catmandu::Importer::MODS;

  my $importer = Catmandu::Importer::MODS->new(file => "modsCollection.xml");

  my $numModsElements = $importer->each(sub{
      my $modsElement = shift; # a MODS::Element::Mods object
  });

MediaMosa

    use Catmandu::Importer::MediaMosa;

    my $importer = Catmandu::Importer::MediaMosa->new(base_url => '...' , user => '...' , password => '...' );

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

MendeleyCatalog

OAI

    # From the command line
    $ catmandu convert OAI --url http://myrepo.org/oai

    $ catmandu convert OAI --url http://myrepo.org/oai --metadataPrefix didl --handler raw

    # In perl
    use Catmandu::Importer::OAI;

    my $importer = Catmandu::Importer::OAI->new(
                    url => "...",
                    metadataPrefix => "..." ,
                    from => "..." ,
                    until => "..." ,
                    set => "...",
                    handler => "..." );

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

PICA

    use Catmandu::Importer::PICA;

    my $importer = Catmandu::Importer::PICA->new(file => "pica.xml", type=> "XML");

    my $n = $importer->each(sub {
        my $hashref = shift;
        # ...
    });

To convert between PICA+ syntax variants with the catmandu command line client:

    catmandu convert PICA --type xml to PICA --type plain < picadata.xml

RDF

Command line client catmandu:

  catmandu convert RDF --url http://d-nb.info/gnd/4151473-7 to YAML

  catmandu convert RDF --file rdfdump.ttl to JSON

  # Query a SPARQL endpoint
  catmandu convert RDF --url http://dbpedia.org/sparql 
                       --sparql "SELECT ?film WHERE { ?film dct:subject <http://dbpedia.org/resource/Category:French_films> }"

  catmandu convert RDF --url http://example.org/sparql --sparql query.rq

  # Query a Linked Data Fragment endpoint
  catmandu convert RDF --url http://fragments.dbpedia.org/2014/en
                       --sparql "SELECT ?film WHERE { ?film dct:subject <http://dbpedia.org/resource/Category:French_films> }"

In Perl code:

    use Catmandu::Importer::RDF;
    my $url = "http://dx.doi.org/10.2474/trol.7.147";
    my $rdf = Catmandu::Importer::RDF->new( url => $url )->first;

RIS

Command line interface:

  catmandu convert RIS < input.txt

  # Use the --human option to translate RIS tags into human readable strings
  catmandu convert RIS --human 1 < input.txt

  # Provide a comma separated mapping file to translate RIS tags
  catmandu convert RIS --human mappings/my_tags.txt < input.txt

In Perl code:

  use Catmandu::Importer::RIS;

  my $importer = Catmandu::Importer::RIS->new(file => "/foo/bar.txt");

  my $n = $importer->each(sub {
    my $hashref = $_[0];
    # ...
  });

SRU

  use Catmandu::Importer::SRU;

  my %attrs = (
    base => 'http://www.unicat.be/sru',
    query => '(isbn=0855275103 or isbn=3110035170 or isbn=9010017362 or isbn=9014026188)',
    recordSchema => 'marcxml',
    parser => 'marcxml'
  );

  my $importer = Catmandu::Importer::SRU->new(%attrs);

  my $count = $importer->each(sub {
        my $schema   = $record->{recordSchema};
        my $packing  = $record->{recordPacking};
        my $position = $record->{recordPosition};
        my $data     = $record->{recordData};
    # ...
  });

  # Using Catmandu::Importer::SRU::Package::marcxml, included in this release

  my $importer = Catmandu::Importer::SRU->new(
    base => 'http://www.unicat.be/sru',
    query => '(isbn=0855275103 or isbn=3110035170 or isbn=9010017362 or isbn=9014026188)',
    recordSchema => 'marcxml' ,
    parser => 'marcxml' ,
  );

  # Using a homemade parser
  
  my $importer = Catmandu::Importer::SRU->new(
    base => 'http://www.unicat.be/sru',
    query => '(isbn=0855275103 or isbn=3110035170 or isbn=9010017362 or isbn=9014026188)',
    recordSchema => 'marcxml' ,
    parser => MyParser->new , # or parser => '+MyParser'
  );

Solr

    use Catmandu::Importer::Solr;

    my %attrs = (
        url => "http://localhost:8983/solr",
        query => 'type:book',
        bag_field => '_bag',
        id_field => '_id'
    );

    my $importer = Catmandu::Importer::Solr->new(%attrs);

    $importer->each(sub {
            my $row_hash = shift;
            ...
    });


    # or

    $ catmandu convert Solr --url "http://localhost:8983/solr" --query "type:book"

Twitter

    use Catmandu::Importer::Twitter;

    my $importer = Catmandu::Importer::Twitter->new(
                        consumer_key => '<your key>' ,
                        consumer_secret => '<your secret>' ,
                        access_token => '<your token>' ,
                        access_token_secret => '<your token secret>' ,    
                        query => '#elag2013' 
                    );

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

Wikidata

    catmandu convert Wikidata --ids Q1,P227
    catmandu convert Wikidata --site dewiki --title Wahnsinn

    echo Q1 | catmandu convert Wikidata
    echo Wahnsinn | catmandu convert Wikidata --site dewiki
    echo dewiki:Wahnsinn | catmandu convert Wikidata

    echo Q1 | catmandu convert Wikidata --fix 'retain_field("labels")'

XLS

    # On the command line
    $ catmandu convert XLS < ./t/test.xls
    $ catmandu convert XLS --header 0 < ./t/test.xls
    $ catmandu convert XLS --fields 1,2,3 < ./t/test.xls
    $ catmandu convert XLS --columns 1 < ./t/test.xls

    # Or in Perl
    use Catmandu::Importer::XLS;

    my $importer = Catmandu::Importer::XLS->new(file => "./t/test.xls");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

XLSX

    # On the command line
    $ catmandu convert XLSX < ./t/test.xlsx
    $ catmandu convert XLSX --header 0 < ./t/test.xlsx
    $ catmandu convert XLSX --fields 1,2,3 < ./t/test.xlsx
    $ catmandu convert XLSX --columns 1 < ./t/test.xlsx

    # Or in Perl
    use Catmandu::Importer::XLSX;

    my $importer = Catmandu::Importer::XLSX->new(file => "./t/test.xlsx");

    my $n = $importer->each(sub {
        my $hashref = $_[0];
        # ...
    });

XML

Z3950

  use Catmandu::Importer::Z3950;

  my %attrs = (
    host => 'z3950.loc.gov',
    port => 7090,
    databaseName => "Voyager",
    preferredRecordSyntax => "USMARC",
    queryType => 'PQF', # CQL or PQF
    query => '@attr 1=4 dinosaur'
  );

  my $importer = Catmandu::Importer::Z3950->new(%attrs);

  my $n = $importer->each(sub {
    my $hashref = $_[0];
    ...
  });

  print "DONE. ($n). \n";

  `host`, `database` and `query` are required, `user` and `user` are optional.
  `port`, `preferredRecordSyntax` and `queryType` will default to 210 'USMARC' and 'CQL' respectively.

Zotero

    # From the command line
    # From the command line
    $ catmandu convert Zotero --userID <userID> to JSON
    $ catmandu convert Zotero --groupID <groupID> to JSON
  
    # From Perl
    use Catmandu;

    my $importer = Catmandu->importer('Zotero', userID => '...');

    $importer->each(sub {
       my $item = shift;
       print "%s %s\n", $item->{_id} , $item->{title}->[0];
   });

Exporter

CSV

    use Catmandu::Exporter::CSV;

    my $exporter = Catmandu::Exporter::CSV->new(
                fix => 'myfix.txt',
                quote_char => '"',
                sep_char => ',',
                escape_char => '"' ,
                always_quote => 1,
                header => 1);

    $exporter->fields("f1,f2,f3");
    $exporter->fields([qw(f1 f2 f3)]);

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    printf "exported %d objects\n" , $exporter->count;

JSON

Command line interface:

    catmandu convert YAML to JSON --pretty 1 < input.yml

In Perl code:

    use Catmandu -all;

    my $exporter = exporter('JSON', fix => 'myfix.txt');

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    printf "exported %d objects\n" , $exporter->count;

Multi

    # this will write both a CSV and an XLS file
    my $exporter = Catmandu::Exporter::Multi->new(exporters => [
        Catmandu::Exporter::CSV->new(file => 'mydata.csv'),
        Catmandu::Exporter::XLS->new(file => 'mydata.xls'),
    ]);
    $exporter->add({col1 => 'val1', col2 => 'val2'});
    $exporter->commit;

Null

        # From the commandline
        $ catmandu convert JSON --fix myfixes to Null < /tmp/data.json

Text

    use Catmandu::Exporter::Text;

    # Print to STDOUT
    my $exporter = Catmandu::Exporter::YAML->new(fix => 'myfix.txt');

    # Print to file or IO::Handle
    my $exporter = Catmandu::Exporter::YAML->new(file => '/tmp/out.yml');
    my $exporter = Catmandu::Exporter::YAML->new(file => $fh);

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    printf "exported %d objects\n" , $exporter->count;

YAML

    use Catmandu::Exporter::YAML;

    # Print to STDOUT
    my $exporter = Catmandu::Exporter::YAML->new(fix => 'myfix.txt');

    # Print to file or IO::Handle
    my $exporter = Catmandu::Exporter::YAML->new(file => '/tmp/out.yml');
    my $exporter = Catmandu::Exporter::YAML->new(file => $fh);

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    printf "exported %d objects\n" , $exporter->count;

Atom

    use Catmandu::Exporter::Atom;

    my $blog_args = {
     id => "urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6" ,
     title => "My Blog" , 
     subtitle => "testing 1.2.3" ,
     icon => "http://icons.org/test.jpg" ,
     generator => "Catmandu::Exporter::Atom" ,
     rights => "Beer license",
     link => [
                {
         'type' => 'text/html' ,
         'rel'  => 'alternate' ,
         'href' => 'http://www.example.com' ,
                 } 
     ],
     author => [ 
               {
         'name' => 'Daffy' ,
         'email' => 'duck@toons.be' ,
               }
     ] ,
     contributor => [
                {
          'name'  => 'Bugs' ,
          'email' => 'bunny@toons.be'
                }
     ],
     ns => {
         'dc' => 'http://purl.org/dc/elements/1.1/',
     },
     'dc:source' => 'test',
    };
    
    my $exporter = Catmandu::Exporter::Atom->new(%$blog_args);

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);
    
    $exporter->add({
     'title'    => 'My Little Pony' ,
     'subtitle' => 'Data testing for you and me' ,
     'content'  => "sdsadas" ,
     'summary'  => 'Brol 123' ,
     'id'       => '1291821827128172817' ,
     'author' => {
        'name' => 'John Doe' ,
        'email' => 'john@farwaway.org' ,
     } ,
     'contributor' => {
        'name' => 'Rabbit, R' ,
        'email' => 'r.rabbit@farwaway.org' ,
        'homepage' => 'http://faraway.org/~rabbit' ,
     } ,
     'link' => [
               {
        'type' => 'text/html' ,
        'rel'  => 'alternate' ,
        'href' => 'http://www.example.com' ,
        'title' => 'Test test' ,
        'length' => '1231' ,
        'hreflang' => 'eng' ,
                } ,
               {
        'type' => 'text/html' ,
        'rel'  => 'alternate' ,
        'href' => 'http://www.example2.com' ,
                }
     ] ,
     'category' => [
                {
        'scheme' => 'http://localhost:8080/roller/adminblog' ,
        'term' => 'Music',
                }
     ] ,
     'rights' => 'Yadadada',
     'dc:subject' => 'Toyz',
    });

    printf "exported %d objects\n" , $exporter->count;

BagIt

   use Catmandu::Exporter::BagIt

   my $exporter = Catmandu::Exporter::BagIt->new(
                            overwrite     => 0 ,
                            skip_manifest => 0,
                  );

   $exporter->add($bagit_record);

   $exporter->commit;

BibTeX

    use Catmandu::Exporter::BibTeX;

    my $exporter = Catmandu::Exporter::BibTeX->new(fix => 'myfix.txt');

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    $exporter->add({
     type    => 'book',
     _citekey => '389-ajk0-1',
     title    => 'the Zen of {CSS} design',
     author   => ['Dave Shea','Molley E. Holzschlag'],
     isbn     => '0-321-30347-4'
    });

    printf "exported %d objects\n" , $exporter->count;

Table

With catmandu command line client:

  echo '{"one":"my","two":"table"} {"one":"is","two":"nice"}' | \ 
  catmandu convert JSON --multiline 1 to Table
  | one | two   |
  |-----|-------|
  | my  | table |
  | is  | nice  |

  catmandu convert CSV to Table --fields id,name --columns ID,Name < sample.csv
  | ID | Name |
  |----|------|
  | 23 | foo  |
  | 42 | bar  |
  | 99 | doz  |

In Perl scripts:

  use Catmandu::Exporter::Table;
  my $exp = Catmandu::Exporter::Table->new;
  $exp->add({ title => "The Hobbit", author => "Tolkien" });
  $exp->add({ title => "Where the Wild Things Are", author => "Sendak" });
  $exp->add({ title => "One Thousand and One Nights" });
  $exp->commit;

  | author  | title                       |
  |---------|-----------------------------|
  | Tolkien | The Hobbit                  |
  | Sendak  | Where the Wild Things Are   |
  |         | One Thousand and One Nights |

MAB2

    use Catmandu::Exporter::MAB2;
 
    my $exporter = Catmandu::Exporter::MAB2->new(file => "mab2.dat", type => "RAW");
    my $data = {
     record => [
        ...
        [245, '1', 'a', 'Cross-platform Perl /', 'c', 'Eric F. Johnson.'],
        ...
        ],
    };
 
    $exporter->add($data);
    $exporter->commit;

MARC

    # From the command line
    $ catmandu convert MARC --type USMARC to MARC --type XML < /foo/bar.mrc

    # From Perl
    use Catmandu;

    my $importer = Catmandu->importer('MARC', file => "/foo/bar.mrc" , type => 'USMARC');
    my $exporter = Catmandu->exporter('MARC', file => "marc.xml", type => "XML" );

    $exporter->add($importer);
    $exporter->commit;

PICA

RDF

In Perl code:

    use Catmandu -all;

    my $exporter = exporter('RDF',
        file => 'export.rdf',
        type => 'XML',
        fix  => 'rdf.fix'
    );

    $exporter->add( $aref ); # pass RDF data in aREF encoding

    $exporter->commit;

RIS

    use Catmandu::Exporter::RIS;

    my $exporter = Catmandu::Exporter::RIS->new(fix => 'myfix.txt');

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    $exporter->add({
     TI   => 'the Zen of CSS design',
     AU   => ['Dave Shea','Molley E. Holzschlag'],
     IS   => '0-321-30347-4'
    });

    printf "exported %d objects\n" , $exporter->count;

Stat

    # Calculate statistics on the availabity of the ISBN fields in the dataset
    cat data.json | catmandu convert -v JSON to Stat --fields isbn

    # Calculate statistics on the uniqueness of ISBN numbers in the dataset
    cat data.json | catmandu convert -v JSON to Stat --fields isbn --values 1

    # Export the statistics as YAML
    cat data.json | catmandu convert -v JSON to Stat --fields isbn --values 1 --as YAML

Template

    use Catmandu::Exporter::Template;

    my $exporter = Catmandu::Exporter::Template->new(
                fix => 'myfix.txt'
                xml => 1,
                template_before => '<path>/header.xml' ,
                template => '<path>/record.xml' ,
                template_after => '<path>/footer.xml' ,
           );

    $exporter->add_many($arrayref);
    $exporter->add_many($iterator);
    $exporter->add_many(sub { });

    $exporter->add($hashref);

    $exporter->commit; # trigger the template_after

    printf "exported %d objects\n" , $exporter->count;

XLS

    # On the command line
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLS --file test.xls
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLS --file test.xls --header 0
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLS --file test.xls --fields a,c
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLS --file test.xls --fields a,c --columns ALPHA,CHARLIE

    # Or in Perl
    use Catmandu::Exporter::XLS;

    my $exporter = Catmandu::Exporter::XLS->new(
                file => 'test.xls',
                fields => 'a,b,c'
                columns => 'ALPHA,BRAVO,CHARLIE'
                header => 1);

    $exporter->add({a => 1, b => 2, c => 3});
    $exporter->add_many($arrayref);

    $exporter->commit;

    printf "exported %d objects\n" , $exporter->count;

XLSX

    # On the command line
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLSX --file test.xlsx
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLSX --file test.xlsx --header 0
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLSX --file test.xlsx --fields a,c
    $ printf "a,b,c\n1,2,3" | catmandu convert CSV to XLSX --file test.xlsx --fields a,c --columns ALPHA,CHARLIE

    # Or in Perl
    use Catmandu::Exporter::XLSX;

    my $exporter = Catmandu::Exporter::XLSX->new(
                file => 'test.xlsx',
                fields => 'a,b,c'
                columns => 'ALPHA,BRAVO,CHARLIE'
                header => 1);

    $exporter->add({a => 1, b => 2, c => 3});
    $exporter->add_many($arrayref);

    $exporter->commit;

    printf "exported %d objects\n" , $exporter->count;

XML

Store

Hash

   use Catmandu::Store::Hash;

   my $store = Catmandu::Store::Hash->new();

   my $obj1 = $store->bag->add({ name => 'Patrick' });

   printf "obj1 stored as %s\n" , $obj1->{_id};

   # Force an id in the store
   my $obj2 = $store->bag->add({ _id => 'test123' , name => 'Nicolas' });

   my $obj3 = $store->bag->get('test123');

   $store->bag->delete('test123');

   $store->bag->delete_all;

   # All bags are iterators
   $store->bag->each(sub { ... });
   $store->bag->take(10)->each(sub { ... });

Multi

AlephX

 use Catmandu::Store::AlephX;

 my $store = Catmandu::Store::AlephX->new(url => 'http://aleph.ugent.be/X' , username => 'XXX' , password => 'XXX');

 $store->bag('usm01')->each(sub {
 });

AlephX

 use Catmandu::Store::AlephX;

 my $store = Catmandu::Store::AlephX->new(url => 'http://aleph.ugent.be/X' , username => 'XXX' , password => 'XXX');

 $store->bag('usm01')->each(sub {
 });

DBI

    use Catmandu::Store::DBI;

    my $store = Catmandu::Store::DBI->new(
        data_source => 'DBI:mysql:database=test', # prefix "DBI:" optional
        username => '', # optional
        password => '', # optional
    );

    my $obj1 = $store->bag->add({ name => 'Patrick' });

    printf "obj1 stored as %s\n" , $obj1->{_id};

    # Force an id in the store
    my $obj2 = $store->bag->add({ _id => 'test123' , name => 'Nicolas' });

    my $obj3 = $store->bag->get('test123');

    $store->bag->delete('test123');

    $store->bag->delete_all;

    # All bags are iterators
    $store->bag->each(sub { ... });
    $store->bag->take(10)->each(sub { ... });

The catmandu command line client can be used like this:

    catmandu import JSON to DBI --data_source SQLite:mydb.sqlite < data.json

FedoraCommons

 use Catmandu::Store::FedoraCommons;

 my $store = Catmandu::Store::FedoraCommons->new(
         baseurl  => 'http://localhost:8080/fedora',
         username => 'fedoraAdmin',
         password => 'fedoraAdmin',
         model    => 'Catmandu::Store::FedoraCommons::DC' # default
 );

 # We use the DC model, lets store some DC
 my $obj1 = $store->bag->add({ 
                    title => ['The Master and Margarita'] , 
                    creator => ['Bulgakov, Mikhail'] }
            );

 printf "obj1 stored as %s\n" , $obj1->{_id};

 # Force an id in the store
 my $obj2 = $store->bag->add({ _id => 'demo:120812' , title => ['The Master and Margarita']  });

 my $obj3 = $store->bag->get('demo:120812');

 $store->bag->delete('demo:120812');

 $store->bag->delete_all;

 # All bags are iterators
 $store->bag->each(sub {  
     my $obj = $_[0];
     my $pid = $obj->{_id};
     my $ds  = $store->fedora->listDatastreams(pid => $pid)->parse_content;
 });
 
 $store->bag->take(10)->each(sub { ... });
 

Solr

    use Catmandu::Store::Solr;

    my $store = Catmandu::Store::Solr->new(url => 'http://localhost:8983/solr' );

    my $obj1 = $store->bag->add({ name => 'Patrick' });

    printf "obj1 stored as %s\n" , $obj1->{_id};

    # Force an id in the store
    my $obj2 = $store->bag->add({ _id => 'test123' , name => 'Nicolas' });

    # send all changes to solr (committed automatically)
    $store->bag->commit;

    #transaction: rollback issued after 'die'
    $store->transaction(sub{
        $bag->delete_all();
        die("oops, didn't want to do that!");
    });

    my $obj3 = $store->bag->get('test123');

    $store->bag->delete('test123');

    $store->bag->delete_all;

    # All bags are iterators
    $store->bag->each(sub { ... });
    $store->bag->take(10)->each(sub { ... });

    # Some stores can be searched
    my $hits = $store->bag->search(query => 'name:Patrick');

CHI

   # From the command line
   $ catmandu export CHI --driver File --root_dir /data to YAML
   $ catmandu import JSON to CHI --driver File --root_dir /data < data.json

   # From perl
   use Catmandu;

   my $store = Catmandu->store('CHI', driver => 'File' , root_dir => '/data');

   $store->bag->each(sub {
        my $item = shift;
        ...
   });

   $store->bag->add({ test => 123 });

CouchDB

    use Catmandu::Store::CouchDB;

    my $store = Catmandu::Store::CouchDB->new;

    my $obj1 = $store->bag->add({ name => 'Patrick' });

    printf "obj1 stored as %s\n" , $obj1->{_id};

    # Force an id in the store
    my $obj2 = $store->bag->add({ _id => 'test123' , name => 'Nicolas' });

    my $obj3 = $store->bag->get('test123');

    $store->bag->delete('test123');

    $store->bag->delete_all;

    # All bags are iterators
    $store->bag->each(sub { ... });
    $store->bag->take(10)->each(sub { ... });

ElasticSearch

    use Catmandu::Store::ElasticSearch;

    my $store = Catmandu::Store::ElasticSearch->new(index_name => 'catmandu');

    my $obj1 = $store->bag->add({ name => 'Patrick' });

    printf "obj1 stored as %s\n" , $obj1->{_id};

    # Force an id in the store
    my $obj2 = $store->bag->add({ _id => 'test123' , name => 'Nicolas' });

    # Commit all changes
    $store->bag->commit;

    my $obj3 = $store->bag->get('test123');

    $store->bag->delete('test123');

    $store->bag->delete_all;

    # All bags are iterators
    $store->bag->each(sub { ... });
    $store->bag->take(10)->each(sub { ... });

    # Some stores can be searched
    my $hits = $store->bag->search(query => 'name:Patrick');

    # Catmandu::Store::ElasticSearch supports CQL...
    my $hits = $store->bag->search(cql_query => 'name any "Patrick"');

Lucy

    use Catmandu::Store::Lucy;

    my $store = Catmandu::Store::Lucy->new(path => '/path/to/index/');

    my $book = $store->bag->add({ title => 'Advanced Perl' });

    printf "book stored as %s\n", $book->{_id};

    $store->bag->commit;

    $bag->get($id);

    # all bags are iterators
    $bag->each(sub { ... });
    $bag->take(10)->each(sub { ... });

    my $hits = $bag->search(query => 'perl');

    # hits is an iterator
    $hits->each(sub {
        say $_[0]->{title};
    });

    $bag->delete($id);
    $bag->delete_by_query(query => 'perl');
    $bag->delete_all;
    $bag->commit;

MongoDB

    # On the command line
    $ catmandu import -v JSON --multiline 1 to MongoDB --database_name bibliography --bag books < books.json
    $ catmandu export MongoDB --database_name bibliography --bag books to YAML
    $ catmandu count MongoDB --database_name bibliography --bag books --query '{"PublicationYear": "1937"}'

    # In perl
    use Catmandu::Store::MongoDB;

    my $store = Catmandu::Store::MongoDB->new(database_name => 'test');

    my $obj1 = $store->bag->add({ name => 'Patrick' });

    printf "obj1 stored as %s\n" , $obj1->{_id};

    # Force an id in the store
    my $obj2 = $store->bag->add({ _id => 'test123' , name => 'Nicolas' });

    my $obj3 = $store->bag->get('test123');

    $store->bag->delete('test123');

    $store->bag->delete_all;

    # All bags are iterators
    $store->bag->each(sub { ... });
    $store->bag->take(10)->each(sub { ... });

    # Search
    my $hits = $store->bag->search(query => '{"name":"Patrick"}');
    my $hits = $store->bag->search(query => '{"name":"Patrick"}' , sort => { age => -1} );
    my $hits = $store->bag->search(query => {name => "Patrick"} , start => 0 , limit => 100);
    
    my $next_page = $hits->next_page;
    my $hits = $store->bag->search(query => '{"name":"Patrick"}' , page => $next_page);

    my $iterator = $store->bag->searcher(query => {name => "Patrick"});