I'd like to do the equivalent of:
foo=$(echo "$foo"|someprogram)
within lua -- ie, I've got a variable containing a bunch of text, and I'd like to run it through a filter (implemented in python as it happens).
Any hints?
Added: would really like to do this without using a temporary file
As long as your Lua supports io.popen, this problem is easy. The solution is exactly as you have outlined, except instead of $(...) you need a function like this one:
function os.capture(cmd, raw)
local f = assert(io.popen(cmd, 'r'))
local s = assert(f:read('*a'))
f:close()
if raw then return s end
s = string.gsub(s, '^%s+', '')
s = string.gsub(s, '%s+$', '')
s = string.gsub(s, '[\n\r]+', ' ')
return s
end
You can then call
local foo = ...
local cmd = ("echo $foo | someprogram"):gsub('$foo', foo)
foo = os.capture(cmd)
I do stuff like this all the time. Here's a related useful function for forming commands:
local quote_me = '[^%w%+%-%=%@%_%/]' -- complement (needn't quote)
local strfind = string.find
function os.quote(s)
if strfind(s, quote_me) or s == '' then
return "'" .. string.gsub(s, "'", [['"'"']]) .. "'"
else
return s
end
end
I stumbled on this post while trying to do the same thing and never found a good solution, see the code below for how I solved my issues. This implementation allows users to access stdin, stdout, stderr and get the return status code. A simple wrapper is called for simple pipe calls.
Brief note:
popenusesforkunder the hood. Fork's documentation states "The child inherits copies of the parent's set of open file descriptors", so what we are doing is creating new file descriptors for the new process to inherit.
require("posix")
--
-- Simple popen3() implementation
--
function popen3(path, ...)
local r1, w1 = posix.pipe()
local r2, w2 = posix.pipe()
local r3, w3 = posix.pipe()
assert((r1 ~= nil or r2 ~= nil or r3 ~= nil), "pipe() failed")
local pid, err = posix.fork()
assert(pid ~= nil, "fork() failed")
if pid == 0 then
posix.close(w1)
posix.close(r2)
posix.dup2(r1, posix.fileno(io.stdin))
posix.dup2(w2, posix.fileno(io.stdout))
posix.dup2(w3, posix.fileno(io.stderr))
posix.close(r1)
posix.close(w2)
posix.close(w3)
local ret, err = posix.execp(path, unpack({...}))
assert(ret ~= nil, "execp() failed")
posix._exit(1)
return
end
posix.close(r1)
posix.close(w2)
posix.close(w3)
return pid, w1, r2, r3
end
--
-- Pipe input into cmd + optional arguments and wait for completion
-- and then return status code, stdout and stderr from cmd.
--
function pipe_simple(input, cmd, ...)
--
-- Launch child process
--
local pid, w, r, e = popen3(cmd, unpack({...}))
assert(pid ~= nil, "filter() unable to popen3()")
--
-- Write to popen3's stdin, important to close it as some (most?) proccess
-- block until the stdin pipe is closed
--
posix.write(w, input)
posix.close(w)
local bufsize = 4096
--
-- Read popen3's stdout via Posix file handle
--
local stdout = {}
local i = 1
while true do
buf = posix.read(r, bufsize)
if buf == nil or #buf == 0 then break end
stdout[i] = buf
i = i + 1
end
--
-- Read popen3's stderr via Posix file handle
--
local stderr = {}
local i = 1
while true do
buf = posix.read(e, bufsize)
if buf == nil or #buf == 0 then break end
stderr[i] = buf
i = i + 1
end
--
-- Clean-up child (no zombies) and get return status
--
local wait_pid, wait_cause, wait_status = posix.wait(pid)
return wait_status, table.concat(stdout), table.concat(stderr)
end
--
-- Example usage
--
local my_in = io.stdin:read("*all")
--local my_cmd = "wc"
--local my_args = {"-l"}
local my_cmd = "spamc"
local my_args = {} -- no arguments
local my_status, my_out, my_err = pipe_simple(my_in, my_cmd, unpack(my_args))
-- Obviously not interleaved as they would have been if printed in realtime
io.stdout:write(my_out)
io.stderr:write(my_err)
os.exit(my_status)
There is nothing in the Lua standard library to allow this.
Here is an in-depth exploration of the difficulties of doing bidirectional communication properly, and a proposed solution:
if possible, redirect one end of the stream (input or output) to a file. I.e.:
fp = io.popen("foo >/tmp/unique", "w") fp:write(anything) fp:close() fp = io.open("/tmp/unique") x = read("*a") fp:close()
You may be interested in this extension which adds functions to the os and io namespaces to make bidirectional communication with a subprocess possible.
Aha, a possibly better solution:
require('posix')
require('os')
require('io')
function splat_popen(data,cmd)
rd,wr = posix.pipe()
io.flush()
child = posix.fork()
if child == 0 then
rd:close()
wr:write(data)
io.flush()
os.exit(1)
end
wr:close()
rd2,wr2 = posix.pipe()
io.flush()
child2 = posix.fork()
if child2 == 0 then
rd2:close()
posix.dup(rd,io.stdin)
posix.dup(wr2,io.stdout)
posix.exec(cmd)
os.exit(2)
end
wr2:close()
rd:close()
y = rd2:read("*a")
rd2:close()
posix.wait(child2)
posix.wait(child)
return y
end
munged=splat_popen("hello, world","/usr/games/rot13")
print("munged: "..munged.." !")
A not very nice solution that avoids a temporary file...
require("io")
require("posix")
x="hello\nworld"
posix.setenv("LUA_X",x)
i=popen('echo "$LUA_X" | myfilter')
x=i.read("*a")
Here is how I solved the problem, it require lua posix.
p = require 'posix'
local r,w = p.pipe()
local r1,w1 = p.pipe()
local cpid = p.fork()
if cpid == 0 then -- child reads from pipe
w:close()
r1:close()
p.dup(r, io.stdin)
p.dup(w1 ,io.stdout)
p.exec('./myProgram')
r:close()
w1:close()
p._exit(0)
else -- parent writes to pipe
IN = r1
OUT = w
end
During myProgram execution, you'l read and write from normal io and after this part of code you just have to write/read on IN and OUT to comunicate with child program.
For a system I have running Lua 5.1 and luaposix 35.0-1, I started with the solution from Anthony Towns from this current page and made it work for this luaposix version for the purpose of calling openssl for encryption on a system without any other encryption capabilities. I have attempted to make the code more explicit in order to allow others to handle any potential API changes in luaposix in the future.
local posix = require('posix');
require('os');
require('io');
local function getOutputFromProcessProvidedInput( dataForProcess, command, commandArguments )
local MAXIMUM_BYTE_READ_COUNT = 100;
local readFileHandle1,writeFileHandle1 = posix.pipe()
io.flush();
local childProcessId1 = posix.fork();
if (childProcessId1 == 0)
then
posix.close( readFileHandle1 );
posix.write( writeFileHandle1, dataForProcess );
io.flush();
os.exit( 1 );
end
posix.close( writeFileHandle1 );
local readFileHandle2,writeFileHandle2 = posix.pipe();
io.flush();
local childProcessId2 = posix.fork();
if (childProcessId2 == 0)
then
posix.close( readFileHandle2 );
posix.dup2( readFileHandle1, posix.fileno( io.stdin ) );
posix.dup2( writeFileHandle2, posix.fileno( io.stdout ) );
posix.execp( command, commandArguments );
os.exit( 2 );
end
posix.close( writeFileHandle2 );
posix.close( readFileHandle1 );
local dataFromProcess = posix.read( readFileHandle2, MAXIMUM_BYTE_READ_COUNT );
posix.close( readFileHandle2 );
posix.wait( childProcessId2 );
posix.wait( childProcessId1 );
return dataFromProcess;
end
-- Command being executed
-- echo -n AAAAAAAAAAAAAAAA | openssl aes-128-cbc -e -nopad -a -K 30313233343536373839616263646566 -iv 1FF1ECB9000000000000000000000000
-- Expected result
-- 28iudIC31lHfDDxfa1/g9w==
result = openReadWritePipe("AAAAAAAAAAAAAAAA","openssl",{"aes-128-cbc", "-e", "-nopad", "-a", "-K", "30313233343536373839616263646566", "-iv", "1FF1ECB9000000000000000000000000"});
print("Result: "..result);
It's easy, no extensions necessary (tested with lua 5.3).
#!/usr/bin/lua
-- use always locals
local stdin = io.stdin:lines()
local stdout = io.write
for line in stdin do
stdout (line)
end
save as inout.lua and do chmod +x /tmp/inout.lua
20:30 $ foo=$(echo "bla"| /tmp/inout.lua)
20:30 $ echo $foo
bla
