AWKing through a XML file
- Relational pipes
- Principles
- Roadmap
- FAQ
- Specification
- Implementation
- Examples
- License
- Screenshots
- Download
- Support & contact
The AWK programming language and tool is designed for processing text but in Relational pipes we can use all the power of AWK for processing relational data that came from various sources. And „all the power“ means that we internally call the regular AWK implementation installed in our system (which is usually GNU AWK).
In this example, we will read a D-Bus policy configuration file (an XML) and filter it using a simple AWK expression. The XML file looks like this:
<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN" "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
<busconfig>
<policy user="root">
<allow own="org.freedesktop.NetworkManager"/>
<allow send_destination="org.freedesktop.NetworkManager"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.PPP"/>
<allow send_interface="org.freedesktop.NetworkManager.SecretAgent"/>
<!-- These are there because some broken policies do
<deny send_interface="..." /> (see dbus-daemon(8) for details).
This seems to override that for the known VPN plugins.
-->
<allow send_destination="org.freedesktop.NetworkManager.openconnect"/>
<allow send_destination="org.freedesktop.NetworkManager.openswan"/>
<allow send_destination="org.freedesktop.NetworkManager.openvpn"/>
<allow send_destination="org.freedesktop.NetworkManager.pptp"/>
<allow send_destination="org.freedesktop.NetworkManager.vpnc"/>
<allow send_destination="org.freedesktop.NetworkManager.ssh"/>
<allow send_destination="org.freedesktop.NetworkManager.iodine"/>
<allow send_destination="org.freedesktop.NetworkManager.l2tp"/>
<allow send_destination="org.freedesktop.NetworkManager.libreswan"/>
<allow send_destination="org.freedesktop.NetworkManager.fortisslvpn"/>
<allow send_destination="org.freedesktop.NetworkManager.strongswan"/>
<allow send_interface="org.freedesktop.NetworkManager.VPN.Plugin"/>
<allow send_destination="org.fedoraproject.FirewallD1"/>
<!-- Allow the custom name for the dnsmasq instance spawned by NM
from the dns dnsmasq plugin to own it's dbus name, and for
messages to be sent to it.
-->
<allow own="org.freedesktop.NetworkManager.dnsmasq"/>
<allow send_destination="org.freedesktop.NetworkManager.dnsmasq"/>
</policy>
<policy user="whoopsie">
<allow send_destination="org.freedesktop.NetworkManager"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.DBus.Introspectable"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.DBus.Properties"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Connection.Active"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device"/>
</policy>
<policy context="default">
<deny own="org.freedesktop.NetworkManager"/>
<deny send_destination="org.freedesktop.NetworkManager"/>
<!-- Basic D-Bus API stuff -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.DBus.Introspectable"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.DBus.Properties"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.DBus.ObjectManager"/>
<!-- Devices (read-only properties, no methods) -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Adsl"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Bond"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Bridge"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Bluetooth"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Wired"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Generic"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Gre"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Infiniband"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Macvlan"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Modem"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.OlpcMesh"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Team"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Tun"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Veth"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Vlan"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.WiMax.Nsp"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.AccessPoint"/>
<!-- Devices (read-only, no security required) -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.WiMax"/>
<!-- Devices (read/write, secured with PolicyKit) -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device.Wireless"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Device"/>
<!-- Core stuff (read-only properties, no methods) -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Connection.Active"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.DHCP4Config"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.DHCP6Config"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.IP4Config"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.IP6Config"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.VPN.Connection"/>
<!-- Core stuff (read/write, secured with PolicyKit) -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Settings"/>
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Settings.Connection"/>
<!-- Agents; secured with PolicyKit. Any process can talk to
the AgentManager API, but only NetworkManager can talk
to the agents themselves. -->
<allow send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.AgentManager"/>
<!-- Root-only functions -->
<deny send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager"
send_member="SetLogging"/>
<deny send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager"
send_member="Sleep"/>
<deny send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Settings"
send_member="LoadConnections"/>
<deny send_destination="org.freedesktop.NetworkManager"
send_interface="org.freedesktop.NetworkManager.Settings"
send_member="ReloadConnections"/>
<deny own="org.freedesktop.NetworkManager.dnsmasq"/>
<deny send_destination="org.freedesktop.NetworkManager.dnsmasq"/>
</policy>
<limit name="max_replies_per_connection">1024</limit>
<limit name="max_match_rules_per_connection">2048</limit>
</busconfig>
We separate particular steps of our pipeline to shell functions for better readability and maintenance:
#!/bin/bash
getXML() {
# Just read a file:
cat "/etc/dbus-1/system.d/org.freedesktop.NetworkManager.conf"
# Or we can wget or curl it from the network
# or generate on-the-fly using some other command.
}
parseXML() {
# Convert the XML tree structure into multiple relations.
# Use XPath expressions to find record nodes
# and to find attributes inside them.
relpipe-in-xmltable \
--relation 'policy' \
--records '/busconfig/policy/allow|/busconfig/policy/deny' \
--attribute 'policy' string 'name()' \
--attribute 'user' string '../@user' \
--attribute 'context' string '../@context' \
--attribute 'own' string '@own' \
--attribute 'send_destination' string '@send_destination' \
--attribute 'send_interface' string '@send_interface' \
--attribute 'send_member' string '@send_member' \
--relation 'limit' \
--records '/busconfig/limit' \
--attribute 'name' string '@name' \
--attribute 'value' integer '.'
}
filterRecords() {
# Use native AWK command (called from relpipe-tr-awk)
# with all its power to filter the records.
# Work with named fields instead of numbered columns.
relpipe-tr-awk \
--relation "policy" \
--where 'policy == "allow" && user != "root"'
# Of course, we can do the same using SQL:
# relpipe-tr-sql \
# --relation "policy" \
# "SELECT * FROM policy WHERE policy = 'allow' AND user <> 'root'"
# --copy 'limit'
# Or using Scheme:
# relpipe-tr-scheme \
# --relation policy \
# --where '(and (string= $policy "allow") (not (string= $user "root")) )'
}
formatOutput() {
relpipe-out-tabular
# Or generate some other format:
# relpipe-out-ods # spreadsheet for e.g. LibreOffice
# relpipe-out-xml # XML e.g. for further XSLT or XQuery processing
# relpipe-out-recfile # recfile for further GNU Recutils processing
# relpipe-out-asn1 # ASN.1 BER for the telco guys
# relpipe-out-gui # display data in a GUI window
# etc.
}
# put the whole pipeline together:
getXML | parseXML | filterRecords | formatOutput
This simple sequence of steps will generate following result:
policy:
╭─────────────────┬───────────────┬──────────────────┬──────────────┬────────────────────────────────┬────────────────────────────────────────────────────┬──────────────────────╮
│ policy (string) │ user (string) │ context (string) │ own (string) │ send_destination (string) │ send_interface (string) │ send_member (string) │
├─────────────────┼───────────────┼──────────────────┼──────────────┼────────────────────────────────┼────────────────────────────────────────────────────┼──────────────────────┤
│ allow │ whoopsie │ │ │ org.freedesktop.NetworkManager │ │ │
│ allow │ whoopsie │ │ │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Introspectable │ │
│ allow │ whoopsie │ │ │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Properties │ │
│ allow │ whoopsie │ │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager │ │
│ allow │ whoopsie │ │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Connection.Active │ │
│ allow │ whoopsie │ │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Introspectable │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Properties │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.ObjectManager │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Adsl │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Bond │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Bridge │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Bluetooth │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Wired │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Generic │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Gre │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Infiniband │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Macvlan │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Modem │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.OlpcMesh │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Team │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Tun │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Veth │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Vlan │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.WiMax.Nsp │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.AccessPoint │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.WiMax │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Wireless │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Connection.Active │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.DHCP4Config │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.DHCP6Config │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.IP4Config │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.IP6Config │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.VPN.Connection │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Settings │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Settings.Connection │ │
│ allow │ │ default │ │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.AgentManager │ │
╰─────────────────┴───────────────┴──────────────────┴──────────────┴────────────────────────────────┴────────────────────────────────────────────────────┴──────────────────────╯
Record count: 39
limit:
╭────────────────────────────────┬─────────────────╮
│ name (string) │ value (integer) │
├────────────────────────────────┼─────────────────┤
│ max_replies_per_connection │ 1024 │
│ max_match_rules_per_connection │ 2048 │
╰────────────────────────────────┴─────────────────╯
Record count: 2
We have processed two relations.
One (limit) was just passed through
while another (policy) was transformed.
In this case, we did just restriction (filtered certain records), but we can do also projections or other relational operations in our pipelines.
This example also shows how we can structure our scripts – compose the pipeline from reusable fragments (shell functions). Once we write the parse function for an input format, we can use it many times, with various filters/transformations and output formatters.
Instead of converting to an output format, we can also execute arbitrary system commands for each record (while having particular attribute values reliably separated in shell variables).
Relational pipes, open standard and free software © 2018-2022 GlobalCode