class LogTools::PostProcessing

exclude(matcher = LogStreamMatcher.new) click to toggle source

Excludes regions of the log file for post-processing

See {include} for more information about inclusions and exclusions

# File lib/log_tools/post_processing.rb, line 369
def exclude(matcher = LogStreamMatcher.new)
    region_matchers << [false, matcher]
    matcher
end

exclude_all() click to toggle source

# File lib/log_tools/post_processing.rb, line 336
def exclude_all
    region_matchers.clear
    @exclude_all = true
end

find_all_included_streams(logfiles) click to toggle source

Gets the set of streams that should be post-processed

@param [{Pocolog::Logfiles=>}] the set of logfiles

along with the set of matchers that should be used to filter this
logfile's streams. This is the same format returned by
{open_logfiles}

@return [{Pocolog::DataStream=>}] the set of streams along

with the set of matchers that should be used to further filter them.

# File lib/log_tools/post_processing.rb, line 61
def find_all_included_streams(logfiles)
    streams = Hash.new
    logfiles.each do |file, matchers|
        file.streams.map do |s|
            if related_matchers = perform_simple_match(!@exclude_all, matchers, "stream", s)
                streams[s] = related_matchers
            end
        end
    end
    streams
end

find_all_logfiles_in_dir(dir) click to toggle source

# File lib/log_tools/post_processing.rb, line 184
def find_all_logfiles_in_dir(dir)
    Dir.enum_for(:glob, File.join(dir, '*.log')).
        find_all { |file| file =~ /\.\d+\.log$/ }
end

find_all_matching_regions(streams, annotations) click to toggle source

Given a set of streams and annotations, outputs a per-stream set of regions that have to be postprocessed

The regions are sorted by time, and are guaranteed to not overlap

@param [{Pocolog::DataStream=>}] streams the set of streams

along with the set of matchers that should be used to determine the
regions. The method performs an union of the returned regions. This
is the same format as the value returned by
{find_all_included_streams}

@param [Array<Orocos::Log::Annotation>] array of annotations sorted in

time

@return [{Pocolog::DataStream=>}] a per-stream list of regions

that are sorted in time, and non-overlapping

# File lib/log_tools/post_processing.rb, line 109
def find_all_matching_regions(streams, annotations)
    regions_by_stream = Hash.new
    streams.each do |s, matchers|
        if s.empty?
            regions_by_stream[s] = []
            next
        end

        initial_region = if @exclude_all then []
                         else [Region.new(*s.time_interval)]
                         end

        raw_regions = matchers.inject(initial_region) do |whole_region, (inclusive, matcher)|
            r = matcher.matching_regions(s, annotations)
            if inclusive
                normalize_regions(whole_region + r)
            else
                substract_regions(whole_region, normalize_regions(r))
            end
        end
        regions_by_stream[s] = raw_regions
    end
    regions_by_stream
end

gather_annotations(streams) click to toggle source

Gets all annotations stored in the provided streams

# File lib/log_tools/post_processing.rb, line 135
def gather_annotations(streams)
    raw_annotations = streams.keys.inject(Array.new) do |a, s|
        if s.type.name == "/logger/Annotations"
            a.concat(s.samples.enum_for(:raw_each).to_a)
        end
        a
    end
    raw_annotations.map(&:last).sort_by { |s| s.raw_time.microseconds }
end

include(matcher = LogStreamMatcher.new) click to toggle source

Includes regions of the log file for post-processing

By default, all the data is included

Inclusions and exclusions are processed in declaration order. This means that e.g. calling {include_logfile} and then {exclude_logstream} will exclude streams from this logfile. The other way around, the log stream exclusion would have no effect on the logfile inclusion (i.e. the log file would be completely included)

@return [LogStreamMatcher]

@example match a log file

processor.include_regions_matching.
    logfile(/camera/)

@example match all Time streams

processor.include_regions_matching.
    stream_type("/base/Time")

# File lib/log_tools/post_processing.rb, line 361
def include(matcher = LogStreamMatcher.new)
    region_matchers << [true, matcher]
    matcher
end

include_all() click to toggle source

# File lib/log_tools/post_processing.rb, line 331
def include_all
    region_matchers.clear
    @exclude_all = false
end

normalize_regions(regions) click to toggle source

Ensures that an array of Region objects is sorted and non-overlapping

@param [Array<Region>] non-empty array of Region @return [Array<Region>]

# File lib/log_tools/post_processing.rb, line 77
def normalize_regions(regions)
    regions = regions.sort_by { |r| r.start_time }
    filtered_regions = Array.new
    last_region = regions.shift
    regions.each do |r|
        if r.start_time < last_region.end_time
            last_region.end_time = [last_region.end_time, r.end_time].max
        else
            filtered_regions << last_region
            last_region = r
        end
    end
    if last_region
        filtered_regions << last_region
    end
    filtered_regions
end

open_logfiles(paths, matchers = region_matchers) click to toggle source

Opens the log files that are included by the configured matchers

@param [String] dir the dataset's directory @return [{Pocolog::Logfiles=>}] the set of opened logfiles

along with the set of matchers that should be used to further filter
the logfile's streams

# File lib/log_tools/post_processing.rb, line 42
def open_logfiles(paths, matchers = region_matchers)
    logfiles = Hash.new
    paths.each do |path|
        basename = File.basename(path)
        if related_matchers = perform_simple_match(!@exclude_all, matchers, "logfile", basename)
            logfiles[Pocolog::Logfiles.open(path)] = related_matchers
        end
    end
    logfiles
end

perform_simple_match(initial_value, matchers, type, object, *args) click to toggle source

# File lib/log_tools/post_processing.rb, line 263
def perform_simple_match(initial_value, matchers, type, object, *args)
    related_matchers = Array.new
    does_match = matchers.inject(initial_value) do |v, (inclusion_matcher, matcher)|
        if !matcher.send("filters_#{type}s?")
            related_matchers << [inclusion_matcher, matcher]
            next(v || inclusion_matcher)
        end

        result = matcher.send("matches_#{type}?", object, *args)
        if result.nil?
            related_matchers << [inclusion_matcher, matcher]
            v || inclusion_matcher
        elsif result
            related_matchers << [inclusion_matcher, matcher]
            inclusion_matcher
        else v
        end
    end
    if does_match
        related_matchers
    end
end

run(dir, target_dir) click to toggle source

# File lib/log_tools/post_processing.rb, line 231
def run(dir, target_dir)
    files = Dir.enum_for(:glob, File.join(dir, '*.*.log')).to_a
    logfiles = open_logfiles(files)
    streams  = find_all_included_streams(logfiles)
    annotations = gather_annotations(streams)
    regions_by_stream = find_all_matching_regions(streams, annotations)

    data_buffer = String.new

    target_logfiles = Hash.new
    regions_by_stream.each do |stream, regions|
        logfile_path = stream.logfile.io.path
        logfile_basename = File.basename(logfile_path).gsub(/\.\d+\.log$/, '')
        logfile = target_logfiles[logfile_basename]
        if !logfile
            target_path = File.join(target_dir, logfile_basename)
            target_logfiles[logfile_basename] = Pocolog::Logfiles.
                create(target_path)
        end

        target_stream = logfile.create_stream(stream.name, stream.type, stream.metadata)
        regions.each do |r|
            header = stream.seek(r.start_time)
            while header && header.lg < r.end_time
                logfile.data(header, data_buffer)
                target_stream.write_raw(header.rt, header.lg, data_buffer)
                header = stream.advance
            end
        end
    end
end

show(dir) click to toggle source

# File lib/log_tools/post_processing.rb, line 196
def show(dir)
    files = find_all_logfiles_in_dir(dir)
    logfiles = open_logfiles(files)
    streams  = find_all_included_streams(logfiles)
    annotations = gather_annotations(streams)
    regions_by_stream = find_all_matching_regions(streams, annotations)

    puts "Available logfiles:"
    files.sort.each do |f|
        puts "  #{f}"
    end
    puts "Matchers:"
    region_matchers.each do |inclusive, m|
        show_matcher(inclusive, m, "  ")
    end
    puts "Matched logfiles:"
    logfiles.each do |file, matchers|
        puts "  #{file.io.first.path}, by"
        matchers.each do |inclusive, m|
            show_matcher(inclusive, m, "    ")
        end
    end
    puts "Matched streams:"
    streams.each do |stream, matchers|
        puts "  #{stream.name}, by"
        matchers.each do |inclusive, m|
            show_matcher(inclusive, m, "    ")
        end
    end
    puts "Processed Regions"
    regions_by_stream.each do |stream, regions|
        puts "  #{stream.name}: #{regions.map { |r| "[#{r.start_time}:#{r.end_time}]" }.join(" ")}"
    end
end

show_matcher(inclusive, m, indent) click to toggle source

# File lib/log_tools/post_processing.rb, line 189
def show_matcher(inclusive, m, indent)
    flag = if inclusive then '+'
           else '-'
           end
    puts "  (#{flag}) #{m}"
end

substract_regions(regions, removed_regions) click to toggle source

Substracts a set of regions from another set

@param [Array<Region>] regions a sorted, non-overlapping list of regions @param [Array<Region>] removed_regions a sorted, non-overlapping list of regions @return [Array<Region>] the set of regions that contain all intervals

in regions with the intervals in removed_regions removed

# File lib/log_tools/post_processing.rb, line 151
def substract_regions(regions, removed_regions)
    regions = regions.dup
    removed_regions = removed_regions.dup

    result = Array.new
    current = regions.shift
    while current && !removed_regions.empty?
        current = current.dup
        current_removal = removed_regions.first
        while current_removal && (current_removal.end_time <= current.start_time)
            current_removal = removed_regions.shift
        end
        if !current_removal
            break
        elsif current.end_time <= current_removal.start_time
            result << current
            current = regions.shift
            next
        end

        if current.start_time < current_removal.start_time
            result << Region.new(current.start_time, current_removal.start_time)
        end

        if current.end_time > current_removal.end_time
            current = Region.new(current_removal.end_time, current.end_time)
        else
            current = regions.shift
        end
    end
    result + [current].compact + regions
end

Attributes

Public Class Methods

Public Instance Methods