class Pocolog::Logfiles

# File lib/pocolog/file.rb, line 219
def self.append(basename)
    io = []
    i = 0
    while File.readable?(path = "#{basename}.#{i}.log")
        io << File.open(path, 'a+')
        i += 1
    end

    if io.empty?
        return create(basename)
    end

    file = Logfiles.new(*io)
    file.basename = basename
    file
end

# File lib/pocolog/convert.rb, line 77
def self.compress(from_io, to_io)
    from = Logfiles.new(from_io)
    from.read_prologue
    write_prologue(to_io, from.endian_swap ^ Pocolog.big_endian?)

    compressed = [1].pack('C')
    buffer = "".encode('BINARY')
    from.each_block(true) do |block_info|
        if block_info.type != DATA_BLOCK || block_info.payload_size < COMPRESSION_MIN_SIZE
            copy_block(block_info, from_io, to_io, buffer)
        else
            # Get the data header
            data_header = from.data_header
            if data_header.compressed
                copy_block(block_info, from_io, to_io, buffer)
                next
            end

            compressed = Zlib::Deflate.deflate(from.data)
            delta = data_header.size - compressed.size
            if Float(delta) / data_header.size > COMPRESSION_THRESHOLD
                # Save it in compressed form
                payload_size = DATA_HEADER_SIZE + compressed.size
                to_io.write([block_info.type, block_info.index, payload_size].pack('CxvV'))
                from_io.seek(block_info.pos + BLOCK_HEADER_SIZE)
                from_io.read(TIME_SIZE * 2, buffer)
                to_io.write(buffer << [compressed.size, 1].pack('VC'))
                to_io.write(compressed)
            else
                copy_block(block_info, from_io, to_io, buffer)
            end
        end
    end
end

# File lib/pocolog/convert.rb, line 12
def self.copy_block(block_info, from_io, to_io, buffer)
    # copy the block as-is
    from_io.seek(block_info.pos)
    buffer = from_io.read(BLOCK_HEADER_SIZE + block_info.payload_size, buffer)
    if block_given?
        buffer = yield(buffer)
    end
    to_io.write(buffer)
end

# File lib/pocolog/file.rb, line 206
def self.create(basename, registry = nil)
    if !registry
        registry = Typelib::Registry.new
        Typelib::Registry.add_standard_cxx_types(registry)
    end
    file = Logfiles.new(registry)
    file.basename = basename
    file.new_file

    file
end

# File lib/pocolog/convert.rb, line 27
def self.from_version_1(from, to_io, big_endian)
    write_prologue(to_io, big_endian)
    from_io = from.rio

    buffer = ""
    uncompressed = [0].pack('C')
    from.each_block(true, false) do |block_info|
        if block_info.type == STREAM_BLOCK
            copy_block(block_info, from_io, to_io, buffer)
        elsif block_info.type == CONTROL_BLOCK
            # remove the fields in time structure
            to_io.write([block_info.type, block_info.index, block_info.payload_size - 16].pack('CxvV'))
            from_io.seek(block_info.pos + BLOCK_HEADER_SIZE + 4)
            to_io.write(from_io.read(8))
            from_io.seek(4, IO::SEEK_CUR)
            to_io.write(from_io.read(1))
            from_io.seek(4, IO::SEEK_CUR)
            to_io.write(from_io.read(8))
        else
            size_offset = - 16 + 1

            to_io.write([block_info.type, block_info.index, block_info.payload_size + size_offset].pack('CxvV'))
            from_io.seek(block_info.pos + BLOCK_HEADER_SIZE + 4)
            to_io.write(from_io.read(8))
            from_io.seek(8, IO::SEEK_CUR)
            to_io.write(from_io.read(8))
            from_io.seek(4, IO::SEEK_CUR)
            to_io.write(from_io.read(4))
            to_io.write(uncompressed)
            from_io.read(block_info.payload_size - (DATA_HEADER_SIZE - size_offset), buffer)
            to_io.write(buffer)
        end
    end
end

# File lib/pocolog/file.rb, line 102
def initialize(*io)
    if io.last.kind_of?(Typelib::Registry)
        @registry = io.pop
    end

    @io          = io
    @io_size     = io.map { |rio| rio.stat.size }
    @streams     = nil
    @block_info  = BlockInfo.new
    @compress    = true
    @data_header_buffer = ""
    rewind
    if io.empty?
        # When opening existing files, @streams is going to be
        # initialized in #streams. However, if we are creating a new set
        # (i.e. io.empty? == true), we also need to tell the system that
        # there currently are no streams available.
        @streams = Array.new
    else
        read_prologue
    end
end

# File lib/pocolog/file.rb, line 831
def self.normalize_metadata(metadata)
    result = Hash.new
    metadata.each do |k, v|
        result[k.to_str] = v
    end
    result
end

# File lib/pocolog/file.rb, line 191
def self.open(pattern, registry = nil)
    io = Dir.enum_for(:glob, pattern).sort.map { |name| File.open(name) }
    if io.empty?
        raise ArgumentError, "no files matching '#{pattern}'"
    end

    if registry
        io << registry
    end
    new(*io)
end

# File lib/pocolog/convert.rb, line 112
def self.rename_streams(from_io, to_io, mappings)
    from = Logfiles.new(from_io)
    from.read_prologue
    write_prologue(to_io, from.endian_swap ^ Pocolog.big_endian?)

    buffer = ""
    from.each_block(true) do |block_info|
        if block_info.type == STREAM_BLOCK
            stream = from.read_stream_declaration
            write_stream_declaration(to_io, stream.index,
                    mappings[stream.name] || stream.name,
                    stream.type, nil, stream.metadata)
        else
            copy_block(block_info, from_io, to_io, buffer)
        end
    end
end

# File lib/pocolog/convert.rb, line 62
def self.to_new_format(from_io, to_io, big_endian = nil)
    from = Logfiles.new(from_io)
    from.read_prologue

rescue MissingPrologue
    # This is format version 1. Need either --little-endian or --big-endian
    if big_endian.nil?
        raise "#{from_io.path} looks like a v1 log file. You must specify either --little-endian or --big-endian"
    end
    puts "#{from_io.path}: format v1 in #{big_endian ? "big endian" : "little endian"}"
    from_version_1(from, to_io, big_endian)

rescue ObsoleteVersion
end

# File lib/pocolog/file.rb, line 76
def self.valid_file?(file)
    Logfiles.new(file)
    true
rescue
    false
end

# File lib/pocolog/file.rb, line 817
def self.write_block(wio, type, index, payload)
    wio << [type, index, payload.size].pack('CxvV')
    wio << payload
    return wio
end

# File lib/pocolog/file.rb, line 953
def self.write_data_block(io, stream_index, rt, lg, compress, data)
    payload = [rt.tv_sec, rt.tv_usec, lg.tv_sec, lg.tv_usec,
        data.length, compress, data
    ].pack("#{DATA_BLOCK_HEADER_FORMAT}a#{data.size}")
    write_block(io, DATA_BLOCK, stream_index, payload)
end

# File lib/pocolog/convert.rb, line 4
def self.write_prologue(to_io, big_endian = nil)
    to_io.write(MAGIC)
    if big_endian.nil?
        big_endian = Pocolog.big_endian?
    end
    to_io.write(*[FORMAT_VERSION, big_endian ? 1 : 0].pack('xVV'))
end

# File lib/pocolog/file.rb, line 840
def self.write_stream_declaration(wio, index, name, type_name, type_registry = nil, metadata = Hash.new)
    if type_name.respond_to?(:name)
        type_registry ||= type_name.registry.minimal(type_name.name).to_xml
        type_name  = type_name.name
    end

    metadata = normalize_metadata(metadata)
    metadata = YAML.dump(metadata)
    payload = [DATA_STREAM, name.size, name, 
        type_name.size, type_name,
        type_registry.size, type_registry,
        metadata.size, metadata
    ].pack("CVa#{name.size}Va#{type_name.size}Va#{type_registry.size}Va#{metadata.size}")
    write_block(wio, STREAM_BLOCK, index, payload)
end

# File lib/pocolog/file.rb, line 360
def advance(index)
    each_data_block(index, false) do
        return data_header
    end
    nil
end

# File lib/pocolog/file.rb, line 130
def close
    io.each { |file| file.close }
end

# File lib/pocolog/file.rb, line 125
def closed?
    @io.all? { |io| io.closed? }
end

# File lib/pocolog/file.rb, line 900
def create_stream(name, type, metadata = Hash.new)
    if type.respond_to?(:to_str)
        type = registry.get(type)
    end

    typename  = type.name
    registry = type.registry.minimal(type.name).to_xml

    @streams ||= Array.new
    new_index = @streams.size
    write_stream_declaration(new_index, name, type.name, registry, metadata)

    stream = DataStream.new(self, new_index, name, typename, registry, metadata)
    @streams << stream
    stream
end

# File lib/pocolog/file.rb, line 787
def data(data_header = nil, buffer = nil)
    if @data && !data_header then @data
    else
        data_header ||= self.data_header
        data_header.io.seek(data_header.payload_pos)
        data = data_header.io.read(data_header.size, buffer)
        if data_header.compressed
            # Payload is compressed
            data = Zlib::Inflate.inflate(data)
        end
        if !data_header
            @data = data
        end
        data
    end
end

# File lib/pocolog/file.rb, line 733
def data_header
    if @data_header.updated
        @data_header
    else
        data_block_pos = rio.tell
        expected_header_size = TIME_SIZE * 2 + 5
        result = rio.read(expected_header_size)
        if result.size != expected_header_size
            raise NotEnoughData, "expected to have #{expected_header_size} bytes remaining, but got only #{@data_header_buffer.size}, you may want to try running pocolog-repair on this file"
        end

        rt_sec, rt_usec, lg_sec, lg_usec, data_size, compressed =
            result.unpack('VVVVVC')
        rt = Time.at(rt_sec, rt_usec)
        lg = Time.at(lg_sec, lg_usec)
        payload_pos = data_block_pos + TIME_SIZE * 2 + 5

        size = payload_pos + data_size - data_block_pos
        expected = block_info.payload_size
        if size != expected
            if rio.respond_to?(:path)
                file = " in #{rio.path}"
            end
            raise NotEnoughData, "payload#{file} at position #{data_block_pos} was expected to be #{expected} bytes, but found #{size}, you may want to try running pocolog-repair on this file"
        end

        @data_header.io  = rio
        @data_header.block_pos   = @block_info.pos
        @data_header.payload_pos = payload_pos
        @data_header.rt = rt
        @data_header.lg = lg
        @data_header.size = data_size
        @data_header.compressed = (compressed != 0)
        @data_header.updated = true
        @data_header
    end
end

# File lib/pocolog/file.rb, line 629
def declared_stream?(index)
    @streams && (@streams.size > index && @streams[index]) 
end

# File lib/pocolog/file.rb, line 317
def each_block(rewind = true, with_prologue = true)
    self.rewind if rewind
    while !eof?
        io = self.rio
        if @next_block_pos == 0 && with_prologue
            read_prologue
        else
            io.seek(@next_block_pos)
        end

        @data = nil
        @data_header.updated = false

        if !read_block_header
            next_io
            next
        end

        yield(@block_info)
    end
rescue EOFError
end

# File lib/pocolog/file.rb, line 436
def each_data_block(stream_index = nil, rewind = true, with_prologue = true)
    each_block(rewind) do |block_info|
        if handle_block(block_info) == DATA_BLOCK
            if !stream_index || stream_index == block_info.index
                yield(block_info.index)
            end
        end
    end

rescue EOFError
rescue
    if !rio
        raise $!
    elsif !rio.closed?
        raise $!, "#{$!.message} at position #{rio.pos}", $!.backtrace
    end
end

# File lib/pocolog/file.rb, line 300
def eof?; @io.size == @rio end

# File lib/pocolog/file.rb, line 306
def file_size; @io_size[@rio] end

# File lib/pocolog/file.rb, line 933
def has_stream?(name)
    !!stream(name)
rescue ArgumentError
end

# File lib/pocolog/file.rb, line 942
def joint_stream(use_rt, *names)
    streams = names.map do |n|
        stream(n)
    end
    JointStream.new(use_rt, *streams)
end

# File lib/pocolog/file.rb, line 496
def load_index_file(index_filename)
    # Look for an index. If it is found, load it and use it.
    return unless File.readable?(index_filename)
    Pocolog.info "loading file info from #{index_filename}... "
    index_data = File.read(index_filename)
    file_info, stream_info =
        begin Marshal.load(index_data)
        rescue Exception => e
            if e.kind_of?(Interrupt)
                raise
            else
                raise InvalidIndex, "cannot unmarshal index data (#{e.message})"
            end
        end

    if file_info.size != @io.size
        raise InvalidIndex, "invalid index file: file set changed"
    end
    coherent = file_info.enum_for(:each_with_index).all? do |(size, _time), idx|
        size == File.size(@io[idx].path)
    end
    if !coherent
        raise InvalidIndex, "invalid index file: file size is different"
    end

    stream_info.each_with_index do |info, idx|
        if(!info.respond_to?("version") || info.version != StreamInfo::STREAM_INFO_VERSION || !info.declaration_block)
            raise InvalidIndex, "old index file found"
        end

        @rio, pos = info.declaration_block
        if read_one_block(pos, @rio).type != STREAM_BLOCK
            raise InvalidIndex, "invalid declaration_block reference in index"
        end

        # Read the stream declaration block and then update the
        # info attribute of the stream object
        if !info.empty?
            @rio, pos = info.interval_io[0]
            if read_one_block(pos, @rio).type != DATA_BLOCK
                raise InvalidIndex, "invalid start IO reference in index"
            end

            if block_info.index != idx
                raise InvalidIndex, "invalid interval_io: stream index mismatch for #{@streams[idx].name}. Expected #{idx}, got #{data_block_index}."
            end

            if !info.index.sane?
                raise InvalidIndex, "index failed internal sanity check"
            end

            @streams[idx].instance_variable_set(:@info, info)
        end
    end
    return @streams.compact

rescue InvalidIndex => e
    Pocolog.warn "invalid index file #{index_filename}: #{e.message}"
    nil
end

# File lib/pocolog/file.rb, line 179
def new_file(filename = nil)
    name = filename || "#{basename}.#{@io.size}.log"
    io = File.new(name, 'w')
    Logfiles.write_prologue(io)
    @io << io
    streams.each_with_index do |s, i|
        write_stream_declaration(i, s.name, s.type.name, registry.to_xml)
    end
end

# File lib/pocolog/file.rb, line 289
def next_io
    @rio += 1
    if @io.size == @rio
        raise EOFError
    else
        read_prologue
        rio
    end
end

# File lib/pocolog/file.rb, line 134
def open
    @io = io.map do |file|
        if file.closed?
            File.open(file.path)
        else file
        end
    end
end

# File lib/pocolog/file.rb, line 346
def read_one_block(pos = nil, rio = nil)
    if pos
        seek(pos, rio)
    end
    each_block(false) do |block_info|
        handle_block(block_info)
        return block_info
    end
    nil
end

# File lib/pocolog/file.rb, line 804
def read_one_data_payload(rio, position, buffer = nil)
    io = @io[rio]
    io.seek(position + BLOCK_HEADER_SIZE + TIME_SIZE * 2)
    data_size, compressed = io.read(5).unpack('VC')
    data = io.read(data_size, buffer)
    if compressed != 0
        # Payload is compressed
        data = Zlib::Inflate.inflate(data)
    end
    data
end

# File lib/pocolog/file.rb, line 247
def rewind
    @rio     = 0
    @time_base      = []
    @time_offset    = []
    @next_block_pos = 0

    @data_header = DataHeader.new
    @data = nil
end

# File lib/pocolog/file.rb, line 302
def rio; @io[@rio] end

# File lib/pocolog/file.rb, line 162
def seek(pos, rio = nil)
    if pos.kind_of?(DataHeader)
        unless io_index = @io.index(pos.io)
            raise "#{pos} does not come from this log fileset"
        end
        @rio = io_index
        @next_block_pos = pos.block_pos
    else
        raise ArgumentError, "need rio argument, if pos is not a DataHeader" unless rio
        @rio = rio
        @next_block_pos = pos
    end
    nil
end

# File lib/pocolog/file.rb, line 922
def stream(name, type = nil, create = false)
    if s = streams.find { |s| s.name == name }
        s.registry # load the registry NOW
        return s
    elsif !type || !create
        raise ArgumentError, "no such stream #{name}"
    end
    create_stream(name, type)
end

# File lib/pocolog/file.rb, line 976
def stream_aligner(use_rt = false)
    StreamAligner.new(use_rt, *streams.compact)
end

# File lib/pocolog/file.rb, line 558
def stream_from_index(index)
    @streams[index]
end

# File lib/pocolog/file.rb, line 887
def stream_from_type(type)
    matches = streams_from_type(type)
    if matches.empty?
        raise ArgumentError, "there is no stream in this file with the required type"
    elsif matches.size > 1
        raise ArgumentError, "there is more than one stream in this file with the required type"
    else
        return matches.first
    end
end

# File lib/pocolog/file.rb, line 564
def streams
    return @streams.compact if @streams

    index_filename = File.basename(@io[0].path, File.extname(@io[0].path)) + ".idx"
    index_filename = File.join(File.dirname(@io[0].path), index_filename)
    if streams = load_index_file(index_filename)
        return streams
    end
    
    # No index file. Compute it.
    Pocolog.info "building index #{index_filename} ..."
    each_data_block(nil, true) do |stream_index|
        # The stream object itself is built when the declaration block
        # has been found
        s    = @streams[stream_index]
        if s.nil? 
            Pocolog.warn "Got empty Streamline. Seems file is corrupted, skipping this" 
        else
            info = s.info
            info.interval_io[1] = [@rio, block_info.pos]
            info.interval_io[0] ||= info.interval_io[1]

            info.index.add_sample_to_index(@rio, data_header.block_pos, data_header.lg)                   
            info.size += 1
        end
    end

    if !@streams
        Pocolog.info "done"
        return []
    end

    @streams.each do |s|
        next unless s

        #set correct time interval
        stream_info = s.info
        if !stream_info.empty?
            @rio, pos = stream_info.interval_io[0]
            rio.seek(pos + BLOCK_HEADER_SIZE)
            stream_info.interval_rt[0] = read_time
            stream_info.interval_lg[0] = read_time
            @rio, pos = stream_info.interval_io[1] || stream_info.interval_io[0]
            rio.seek(pos + BLOCK_HEADER_SIZE)
            stream_info.interval_rt[1] = read_time
            stream_info.interval_lg[1] = read_time          
        end
    end

    file_info   = @io.map { |io| [File.size(io.path), io.mtime] }
    stream_info = @streams.compact.map { |s| s.info }

    begin
        File.open(index_filename, 'w') do |io|
            Marshal.dump([file_info, stream_info], io)
        end
    rescue
        FileUtils.rm_f index_filename
        raise
    end
    Pocolog.info "done"
    @streams.compact
end

# File lib/pocolog/file.rb, line 875
def streams_from_type(type)
    if type.respond_to?(:name)
        type = type.name
    end

    streams.find_all { |s| s.type.name == type }
end

# File lib/pocolog/file.rb, line 771
def sub_field(offset, size, data_header = nil)
    data_header ||= self.data_header
    if data_header.compressed
        raise "field access on compressed files is unsupported"
    end
    data_header.io.seek(data_header.payload_pos + offset)
    data = data_header.io.read(size)
    data
end

# File lib/pocolog/file.rb, line 152
def tell; @next_block_pos end

# File lib/pocolog/file.rb, line 635
def time_base
    if @time_base.empty?
        Time.at(0)
    else
        @time_base.last[1] 
    end
end

# File lib/pocolog/file.rb, line 644
def time_offset
    if @time_offset.empty? then 0
    else @time_offset.last[1] 
    end
end

# File lib/pocolog/file.rb, line 781
def validate_data_block
    data_header = self.data_header
    data_header.io.seek(data_header.payload_pos + data_header.size)
end

# File lib/pocolog/file.rb, line 304
def wio; @io.last end

# File lib/pocolog/file.rb, line 827
def write_block(type,index,payload)
    return Logfiles.write_block(wio,type,index,payload)
end

# File lib/pocolog/file.rb, line 867
def write_stream_declaration(index, name, type, registry, metadata)
    do_write do
        Logfiles.write_stream_declaration(wio, index, name, type, registry, metadata)
    end
end