string_exporter

Processors

string

String Processor to export the content to a String.

Properties

Name	Default Value	Type	Description
sample_separator		string	Sample separator.

Plugin Attributes

Attribute	Value
Plugin Name	string_exporter.adtffileplugin
License	MPL-2.0
Support Mail	suppo.nosp@m.rt@d.nosp@m.igita.nosp@m.lwer.nosp@m.k.net
Homepage URL	https://adtf.dev/

Usage

adtfdat_tool --plugin string_exporter.adtffileplugin --export <adtfdat> --stream <name> --processorid string --output <string>

Source

 
#pragma once
 
#include <adtfdat_processing/processor.h>
#include <memory>
#include <fstream>
#include <ddl/codec/codec_factory.h>
 
enum class EndianessType
{
    unknown_endian = 0x00,
    little_endian = 0x01,
    big_endian = 0x02,
};
 
enum class EncodingType
{
    unknown = 0,
    unspecified = 1,
    c_char = 2,
    utf8 = 3,
    w_char = 4,
    w_char_16 = w_char,
    utf16 = 5,
    utf32 = 6
};
 
class StringExporter : public adtfdat_processing::SingleStreamProcessor
{
public:
    StringExporter();
    ~StringExporter() = default;
 
    std::string getProcessorIdentifier() const override
    {
        return "string";
    }
 
    bool isCompatible(const adtf_file::Stream& stream) const override;
    void open(const adtf_file::Stream& stream, const std::string& destination_url) override;
    void process(const adtf_file::FileItem& item) override;
 
private:
    void setEncodingAndEndianess(std::shared_ptr<const adtf_file::DefaultStreamType> stream_type);
 
private:
    std::string sample_separator;
    EndianessType endianess;
    EncodingType encoding;
    std::ofstream output_file_stream;
    std::ostream* output_stream;
};

 
#include "string_exporter.h"
#include <adtf_file/stream_type.h>
#include <adtf_file/sample.h>
#include <string>
#include <adtfdat_processing/ddl_helpers.h>
#include <iostream>
#include <cuchar>
#include <cstdlib>
 
using namespace std::literals;
 
static adtf_file::PluginInitializer initializer([] {
    adtf_file::getObjects().push_back(
        std::make_shared<adtfdat_processing::ProcessorFactoryImplementation<StringExporter>>());
});
 
EndianessType StringToEndianType(const std::string& endianess)
{
    try {
        switch(static_cast<EndianessType>(std::stoi(endianess)))
        {
            case EndianessType::big_endian:
                return EndianessType::big_endian;
            case EndianessType::little_endian:
                return EndianessType::little_endian;
            default:
                break;
        }
    } catch (...) {
    }
 
    return EndianessType::unknown_endian;
}
 
static const std::vector<std::pair<std::string, std::string>> escape_sequences =
{
    { "\\0"s, "\0"s },
    { "\\n"s, "\n"s },
    { "\\r"s, "\r"s },
    { "\\t"s, "\t"s },
};
 
static void string_replace(std::string& str, const std::string& match, const std::string& replacement) noexcept
{
    size_t index = 0;
    while (true)
    {
        index = str.find(match, index);
        if (index == std::string::npos) {
            break;
        }
 
        str.replace(index, match.size(), replacement);
 
        index += replacement.size();
    }
}
 
StringExporter::StringExporter()
{
    FILE_PLUGIN_META_INFORMATION(BUILD_VERSION, "String Processor to export the content to a String."s);
 
    setConfiguration(
        adtf_file::PropertyBuilder("sample_separator"s, "\\n"s)
            .setDescription("Sample separator."s)
            .setValueList( { { "\\n"s, "New Line (`\\n`)"s },
                             { "\\r"s, "Carriage return (`\\r`)"s },
                             { "\\r\\n"s, "Carriage return and new line (`\\r\\n`)"s },
                             { "\\t"s, "Tabulator (`\\t`)"s },
                             { "\\0"s, "NULL-Byte (`\\0`)."s },
                             { ""s, "No Separator (``)"s } }, false)
    );
}
 
bool StringExporter::isCompatible(const adtf_file::Stream& stream) const
{
    const auto tmp = std::dynamic_pointer_cast<const adtf_file::DefaultStreamType>(stream.initial_type);
    if (tmp)
    {
        if (0 == tmp->getMetaType().compare("adtf/string"))
        {
            return true;
        }
    }
    return false;
}
 
void StringExporter::open(const adtf_file::Stream& stream, const std::string& destination_url)
{
    const auto config = getConfiguration();
    sample_separator = adtf_file::getPropertyValue<std::string>(config, "sample_separator"s);
 
    for(const auto& escape_sequence : escape_sequences)
    {
        string_replace(sample_separator, escape_sequence.first, escape_sequence.second);
    }
 
    const auto stream_type = std::dynamic_pointer_cast<const adtf_file::DefaultStreamType>(stream.initial_type);
    setEncodingAndEndianess(stream_type);
 
    if (destination_url.empty())
    {
        output_stream = &std::cout;
        return;
    }
 
    try
    {
        output_file_stream.open(destination_url);
        output_stream = &output_file_stream;
 
    }
    catch (...)
    {
        std::throw_with_nested(std::runtime_error("unable to open output file '" +
                                                  destination_url + "' for stream " + stream.name));
    }
}
 
EndianessType system_endianess() {
    constexpr uint16_t test_value = 0x0102;
    return (*reinterpret_cast<const uint8_t*>(&test_value) == 0x02) ? EndianessType::little_endian : EndianessType::big_endian;
}
 
static inline char16_t swap_endian(const char16_t& c) noexcept
{
#if defined(__llvm__) || defined(__GNUC__)
    return __builtin_bswap16(c);
#elif defined(_MSC_VER)
    return _byteswap_ushort(c);
#else
    return (c >> 8) | (c << 8);
#endif
}
 
static inline char32_t swap_endian(const char32_t& c) noexcept
{
#if defined(__llvm__) || defined(__GNUC__)
    return __builtin_bswap32(c);
#elif defined(_MSC_VER)
    return _byteswap_ulong(c);
#else
    static_assert(false, "Missing intrinsic for this compiler?");
#endif
}
 
static inline void u16_to_utf8(std::ostream& output, const std::u16string_view& u16str, EndianessType input_endian) noexcept
{
    auto swap_needed = false;
    if (input_endian != EndianessType::unknown_endian)
    {
        swap_needed = (input_endian != system_endianess());
    }
 
    std::mbstate_t state{};
    for (char16_t c : u16str) {
        if (swap_needed) {
            c = swap_endian(c);
        }
 
        // You are supposed to use MB_CUR_MAX - but it's broken on some platforms. So instead we just take the known
        // worst-case of UTF32 to UTF8 which is 6 bytes.
        std::array<char, 6> out;
 
        const auto result = std::c16rtomb(out.data(), c, &state);
        if (result == static_cast<size_t>(-1)) {
            continue;
        }
 
        output << std::string_view(out.data(), result);
    }
}
 
static inline void u32_to_utf8(std::ostream& output, const std::u32string_view& u32str, EndianessType input_endian) noexcept
{
    auto swap_needed = false;
    if (input_endian != EndianessType::unknown_endian)
    {
        swap_needed = (input_endian != system_endianess());
    }
 
    std::mbstate_t state{};
    for (char32_t c : u32str)
    {
        if (swap_needed)
        {
            c = swap_endian(c);
        }
 
        // You are supposed to use MB_CUR_MAX - but it's broken on some platforms. So instead we just take the known
        // worst-case of UTF32 to UTF8 which is 6 bytes.
        std::array<char, 6> out;
        const auto result = std::c32rtomb(out.data(), c, &state);
        if (result == static_cast<size_t>(-1))
        {
            continue;
        }
 
        output << std::string_view(out.data(), result);
    }
}
 
void StringExporter::process(const adtf_file::FileItem& item)
{
    if (const auto sample = std::dynamic_pointer_cast<const adtf_file::WriteSample>(item.stream_item))
    {
        switch(encoding)
        {
            case EncodingType::unknown:
            case EncodingType::c_char:
            case EncodingType::utf8:
            {
                auto buffer = sample->beginBufferRead();
                const std::string_view str(reinterpret_cast<const char*>(buffer.first), buffer.second);
                *output_stream << str << sample_separator;
                break;
            }
            case EncodingType::w_char_16:
            {
                auto buffer = sample->beginBufferRead();
                const std::u16string_view u16str(reinterpret_cast<const char16_t*>(buffer.first),
                                                 buffer.second / sizeof(char16_t));
 
                if (endianess != EndianessType::unknown_endian && endianess != system_endianess())
                {
                    for (const auto& c16 : u16str)
                    {
                        *output_stream << static_cast<char>(c16 >> 8);
                    }
                }
                else
                {
                    *output_stream << std::string(u16str.begin(), u16str.end());
                }
                *output_stream << sample_separator;
                break;
            }
            case EncodingType::utf16:
            {
                auto buffer = sample->beginBufferRead();
                const std::u16string_view u16str(reinterpret_cast<const char16_t*>(buffer.first), buffer.second / sizeof(char16_t));
                u16_to_utf8(*output_stream, u16str, endianess);
                *output_stream << sample_separator;
                break;
            }
            case EncodingType::utf32:
            {
                auto buffer = sample->beginBufferRead();
                const std::u32string_view u32str(reinterpret_cast<const char32_t*>(buffer.first),
                                                 buffer.second / sizeof(char32_t));
                u32_to_utf8(*output_stream, u32str, endianess);
                *output_stream << sample_separator;
                break;
            }
            default:
                break;
        }
    }
    else if(auto stream_type = std::dynamic_pointer_cast<const adtf_file::DefaultStreamType>(item.stream_item))
    {
        setEncodingAndEndianess(stream_type);
    }
}
 
void StringExporter::setEncodingAndEndianess(std::shared_ptr<const adtf_file::DefaultStreamType> stream_type)
{
    if (stream_type)
    {
        endianess = StringToEndianType(stream_type->getProperty("data_endianess").second);
 
        const auto strEncoding = stream_type->getProperty("encoding").second;
 
        if ("c-char"s == strEncoding)
        {
            encoding = EncodingType::c_char;
        }
        else if ("utf8"s == strEncoding)
        {
            encoding = EncodingType::utf8;
        }
        else if ("w-char"s == strEncoding)
        {
            encoding = EncodingType::w_char_16;
        }
        else if ("utf16"s == strEncoding)
        {
            encoding = EncodingType::utf16;
        }
        else if ("utf32"s == strEncoding)
        {
            encoding = EncodingType::utf32;
        }
        else
        {
            encoding = EncodingType::unknown;
        }
    }
}