Skip to content

dli-process-synthetic-syslogs

Pipeline: dli-process-synthetic-syslogs

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
@c:new-block

    ## Create Log Archive repository
    --> @dm:create-logarchive-repo
            repo = "demo_logarchive" &
            prefix = "demo_logs/" &
            retention = 31

--> @c:new-block

    ## Create Persistent Stream with 31 day retention (if not already created)
    --> @dm:create-persistent-stream
            name = "dli-synthetic-logs-processed" &
            retention_days = 31

--> @c:new-block

    ## Create Persistent Stream with 1 day retention for dropped events (for inspection)
    --> @dm:create-persistent-stream
            name = "dli-synthetic-logs-dropped" &
            retention_days = 1

--> @c:new-block

    --> @dm:create-persistent-stream
            name = "dli-log-stats" &
            retention_days = 90

--> @c:new-block

    --> @dm:empty

    --> @dm:addrow
            rule_id = "A100" &
            rule = 'message contains "Unable to convert Vigor value"' &
            reason = "too frequent, does not affect anything"
    --> @dm:addrow
            rule_id = "A101" &
            rule = 'message contains "Re-check service health since"' &
            reason = "too frequent, does not affect anything"
    --> @dm:addrow
            rule_id = "A102" &
            rule = 'message contains "Hostd: verbose "' &
            reason = "Verbose message"
    --> @dm:addrow
            rule_id = "A103" &
            rule = 'message contains "Vpxa: verbose "' &
            reason = "Verbose message"
    --> @dm:addrow
            rule_id = "A104" &
            rule = 'message contains "Rhttpproxy: verbose"' &
            reason = "Verbose message"
    --> @dm:addrow
            rule_id = "A105" &
            rule = 'message contains "Configured from vty by admin"' &
            reason = "Console message"
    --> @dm:addrow
            rule_id = "A106" &
            rule = 'message contains "updatemgr.*DEBUG.*The number of tasks.*"' &
            reason = "too frequent, does not affect anything"
    --> @dm:eval
            action = "'DROP'"
    --> @dm:addrow
            rule_id = "Z999" &
            rule = '*' &
            action = 'KEEP'
    --> @dm:save
            name = "temp-filter-rules"

--> @c:new-block

    ## rn:read-stream bot is Streaming Bot which means this block acts like a loop
    --> @rn:read-stream
            name = "dli-synthetic-logs-raw" &
            group = "demo-log-processor"
    ## save all logs to S3 like storage.
    --> @dm:logarchive-save
            repo = "demo_logarchive" &
            archive = "sythentic-syslogs"
    --> @rn:write-stats-to-stream
            name = "dli-log-stats" &
            groupby = "device" &
            mode = "archived"
    ## Now tag each event using a rules dictionary
    --> @dm:enrich-using-rule-dict
            dict = "temp-filter-rules" &
            enrich_columns = "action,reason"
    --> @dm:save
            name = "temp-tagged-events"
    --> *dm:filter
            action is not 'DROP'
    ## now send the filtered logs to different stream
    --> @rn:write-stream
            name = "dli-synthetic-logs-processed"
    --> @rn:write-stats-to-stream
            name = "dli-log-stats" &
            groupby = "device" &
            mode = "processed"
    --> @dm:recall
            name = "temp-tagged-events"
    --> *dm:filter
            action is 'DROP'
    --> @rn:write-stream
            name = "dli-synthetic-logs-dropped"
    --> @rn:write-stats-to-stream
            name = "dli-log-stats" &
            groupby = "device" &
            mode = "dropped"

Playground

Try this in RDA Playground


Extensions used in this Pipeline

Source Name Extension Type
rn rn


Artifacts used in this Pipeline

Artifact Type Artifact Name Access
rda-network-stream dli-synthetic-logs-raw read
credential demo_logarchive write
rda-network-stream dli-log-stats write
rda-network-stream dli-synthetic-logs-processed write
rda-network-stream dli-synthetic-logs-dropped write


Bots used in this Pipeline

@c:new-block @dm:create-logarchive-repo @dm:create-persistent-stream @dm:empty @dm:addrow @dm:eval @dm:save @rn:read-stream @dm:logarchive-save @rn:write-stats-to-stream @dm:enrich-using-rule-dict *dm:filter @rn:write-stream @dm:recall