PenTest/nmap
1
0
Fork 0
mirror of https://github.com/nmap/nmap.git synced 2025-12-06 12:41:29 +00:00
Code Issues Projects Releases Wiki Activity

Add structured output to http-get.nse.

Browse Source 
This patch is by Alex Weber.
http://seclists.org/nmap-dev/2012/q4/434
This commit is contained in:
david
2012-12-13 04:16:01 +00:00
parent a8ed9f57f8
commit 43f28b8f4a
2 changed files with 182 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG
View File

@@ -1,5 +1,7 @@
# Nmap Changelog ($Id$); -*-text-*- # Nmap Changelog ($Id$); -*-text-*-
o [NSE] Added structured output to http-git.nse. [Alex Weber]
o [NSE] Added murmur-version by Marin Maržić. This gets teh server o [NSE] Added murmur-version by Marin Maržić. This gets teh server
version and other information for Murmur, the server for the Mumble version and other information for Murmur, the server for the Mumble
VoIP system. VoIP system.

316
scripts/http-git.nse
View File

@@ -1,43 +1,70 @@
-- Checks for a Git repository found in a website's document root (GET /.git/<something> HTTP/1.1)
-- Gets as much information about the repository as possible, including language/framework, Github
-- username, last commit message, and repository description.
--
-- @output
-- PORT STATE SERVICE
-- 80/tcp open http
-- | http-git:
-- | Git repository found in web root
-- | Last commit message: This is my last commit...
-- | Repository description: Unnamed repository; edit this file 'description' to name the...
-- | GitHub remote: AlexWebr/nse (accessed over SSH)
-- | BitBucket remote: AlexWebr/nse (accessed over HTTP, pull-only)
-- |_ Based on the file '.gitignore', this is a Ruby on Rails application
--
-- Version 1.1
-- Created 27 June 2012 - written by Alex Weber <alexwebr@gmail.com>
local http = require("http") local http = require("http")
local shortport = require("shortport") local shortport = require("shortport")
local stdnse = require("stdnse") local stdnse = require("stdnse")
local strbuf = require("strbuf") local strbuf = require("strbuf")
local string = require("string") local string = require("string")
local table = require("table") local table = require("table")
description = [[ Checks for a Git repository found in a website's document root (/.git/<something>) then retrieves as much repo information as possible, including language/framework, Github username, last commit message, and repository description.
description = [[
Checks for a Git repository found in a website's document root
/.git/<something>) and retrieves as much repo information as
possible, including language/framework, remotes, last commit
message, and repository description.
]] ]]
categories = { "safe", "vuln", "default" } -- @output
-- PORT STATE SERVICE REASON
-- 80/tcp open http syn-ack
-- | http-git:
-- | 127.0.0.1:80/.git/
-- | Git repository found!
-- | .git/config matched patterns 'passw'
-- | Repository description: Unnamed repository; edit this file 'description' to name the...
-- | Remotes:
-- | http://github.com/someuser/somerepo
-- | Project type: Ruby on Rails web application (guessed from .git/info/exclude)
-- | 127.0.0.1:80/damagedrepository/.git/
-- |_ Potential Git repository found (found 2/6 expected files)
--
-- @xmloutput
-- <table key="127.0.0.1:80/.git/">
-- <table key="remotes">
-- <elem>http://github.com/anotherperson/anotherepo</elem>
-- </table>
-- <table key="project-type">
-- <table key=".git/info/exclude">
-- <elem>JBoss Java web application</elem>
-- <elem>Java application</elem>
-- </table>
-- </table>
-- <elem key="repository-description">A nice repository</elem>
-- <table key="files-found">
-- <elem key=".git/COMMIT_EDITMSG">false</elem>
-- <elem key=".git/info/exclude">true</elem>
-- <elem key=".git/config">true</elem>
-- <elem key=".git/description">true</elem>
-- <elem key=".gitignore">false</elem>
-- </table>
-- <table key="interesting-matches">
-- <table key=".git/config">
-- <elem>passw</elem>
-- </table>
-- </table>
-- </table>
categories = { "default", "safe", "vuln" }
author = "Alex Weber" author = "Alex Weber"
license = "Same as Nmap--See http://nmap.org/book/man-legal.html" license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
portrule = shortport.http portrule = shortport.http
local STATUS_OK = 200 -- We consider 200 to mean "okay, file exists and we received its contents" -- We consider 200 to mean "okay, file exists and we received its contents".
local STATUS_OK = 200
-- Long strings (like a repository's description) will be truncated to this
-- number of characters in normal output.
local TRUNC_LENGTH = 60
function action(host, port) function action(host, port)
-- All methods that we call on this table will be from the table library local out
local out = {}
setmetatable(out, {__index = table})
local repos_found = 0
-- We can accept a single root, or a table of roots to try -- We can accept a single root, or a table of roots to try
local root_arg = stdnse.get_script_args("http-git.root") local root_arg = stdnse.get_script_args("http-git.root")
@@ -51,7 +78,7 @@ function action(host, port)
end end
-- Try each root in succession -- Try each root in succession
for i, root in ipairs(roots) do for _, root in ipairs(roots) do
root = tostring(root) root = tostring(root)
root = root or '/' root = root or '/'
@@ -68,10 +95,7 @@ function action(host, port)
-- We could try for /.git/, but we will not get a 200 if directory -- We could try for /.git/, but we will not get a 200 if directory
-- listings are disallowed. -- listings are disallowed.
if http.get(host, port, root .. ".git/HEAD").status == STATUS_OK then if http.get(host, port, root .. ".git/HEAD").status == STATUS_OK then
if repos_found > 0 then out = out or {}
out:insert("")
end
repos_found = repos_found + 1
local replies = {} local replies = {}
-- This function returns true if we got a 200 OK when -- This function returns true if we got a 200 OK when
-- fetching 'filename' from the server -- fetching 'filename' from the server
@@ -83,111 +107,66 @@ function action(host, port)
-- These files are created by creating and using the repository, -- These files are created by creating and using the repository,
-- or by popular development frameworks. -- or by popular development frameworks.
local repo = { local repo = {
".gitignore",
".git/COMMIT_EDITMSG",
".git/config", ".git/config",
".git/description", ".git/description",
".git/info/exclude", ".git/info/exclude",
".git/COMMIT_EDITMSG",
".gitignore",
} }
local count = { ok = 0, tried = 0 } local pl_requests = {} -- pl_requests = pipelined requests (temp)
local prequests = {} -- prequests = pipelined requests (temp)
-- Go through all of the filenames and do an HTTP GET -- Go through all of the filenames and do an HTTP GET
for _, name in ipairs(repo) do -- for every filename for _, name in ipairs(repo) do -- for every filename
http.pipeline_add(root .. name, nil, prequests) http.pipeline_add(root .. name, nil, pl_requests)
end end
-- do the requests -- Do the requests.
replies = http.pipeline_go(host, port, prequests) replies = http.pipeline_go(host, port, pl_requests)
if replies == nil then if replies == nil then
stdnse.print_debug("%s: pipeline_go() error. Aborting.", SCRIPT_NAME) stdnse.print_debug("%s: pipeline_go() error. Aborting.", SCRIPT_NAME)
return nil return nil
end end
for i, reply in ipairs(replies) do for i, reply in ipairs(replies) do
count.tried = count.tried + 1
-- We want this to be indexed by filename, not an integer, so we convert it -- We want this to be indexed by filename, not an integer, so we convert it
-- We added to the pipeline in the same order as the filenames, so this is safe -- We added to the pipeline in the same order as the filenames, so this is safe.
replies[repo[i]] = reply -- create index by filename replies[repo[i]] = reply -- create index by filename
replies[i] = nil -- delete integer-indexed entry replies[i] = nil -- delete integer-indexed entry
if reply.status == STATUS_OK then count.ok = count.ok + 1 end
end end
-- Tell the user that we found a repository, and indicate if -- Mark each file that we tried to get as 'found' (true) or 'not found' (false).
-- we didn't find all the files we were looking for.
local location = host.ip .. ":" .. port.number .. root .. ".git/" local location = host.ip .. ":" .. port.number .. root .. ".git/"
if count.ok == count.tried then out[location] = {}
out:insert("Git repository found at " .. location) -- A nice shortcut
else -- if we didn't find all the files we were hoping to, we might not actually have a repo local loc = out[location]
out:insert("Potential Git repository found at " .. location .. " (found " .. loc["files-found"] = {}
tostring(count.ok + 1) .. " of " .. tostring(count.tried + 1) .. " expected files)") for name, _ in pairs(replies) do
-- we already got /.git/HEAD, so add one to 'found' and 'expected' loc["files-found"][name] = ok(name)
end end
-- This function matches a table of words/regexes against a single string -- Look through all the repo files we grabbed and see if we can find anything interesting.
-- This function is used immediately after it is declared local interesting = { "bug", "key", "passw", "pw", "user", "secret", "uid" }
local function match_many(str, table_of_words) for name, reply in pairs(replies) do
local matched_string, lstr, t_to_return = false, string.lower(str), {} if ok(name) then
for i, word in ipairs(table_of_words) do for _, pattern in ipairs(interesting) do
matched_string = string.match(lstr, word) if string.match(reply.body, pattern) then
if matched_string then table.insert(t_to_return , matched_string) end -- A Lua idiom - don't create this table until we actually have something to put in it
end loc["interesting-matches"] = loc["interesting-matches"] or {}
return t_to_return loc["interesting-matches"][name] = loc["interesting-matches"][name] or {}
end table.insert(loc["interesting-matches"][name], pattern)
end
-- Look through all the repo files we grabbed and see if we can find anything interesting
local interesting = { "bug", "passw", "pw", "user", "uid", "key", "secret" }
for name, reply in pairs(replies) do
if ok(name) then -- for all replies that were successful
local found_anything = false -- have we found anything yet?
local buf = strbuf.new()
for _, matched in ipairs(match_many(reply.body:lower(), interesting)) do -- match all files against 'interesting'
if not found_anything then -- if this is our first find, print filename and stuff
buf = (((((buf .. "Contents of '") .. name) .. "' matched patterns '") .. matched) .. "'") -- the '..' is right-associative :(
found_anything = true
else
buf = ((buf .. ", '" .. matched) .. "'")
end -- if we found something already, tack this pattern onto the end
end -- If we matched anything, finish the line up
if found_anything then
buf = buf .. " (case-insensitive match)"
out:insert(strbuf.dump(buf))
end end
end end
end end
-- Given a longer plain-text string (a large text file, for example), append
-- a summary of it (the first 60 characters or the first line, whichever is shorter)
local function append_short_version(description, original_string)
local short = string.sub(original_string, 1, 60) -- trim the string first, in case it is huge
-- We try to cut off the newline if we can
local no_newline = string.match(short, "(.-)\r?\n") -- we don't want such an open-ended regex on a potentially huge string
local s = no_newline or short
if #original_string > #s then
-- If we cut off something, we want to put an ellipsis on the end
s = description .. ": " .. s .. "..."
else
s = description .. ": " .. s
end
out:insert(s)
end
-- Get the first line and trim to 60 characters, if we got a COMMIT_EDITMESSAGE
if ok(".git/COMMIT_EDITMSG") then if ok(".git/COMMIT_EDITMSG") then
-- If there's no newline in the file (there usually is), this won't work. loc["last-commit-message"] = replies[".git/COMMIT_EDITMSG"].body
-- Normally, it's a commit message followed by a bunch of commented-out lines (#).
append_short_version("Last commit message", replies[".git/COMMIT_EDITMSG"].body)
end end
-- Do the same thing as for description - get first line, truncate to 60 characters.
if ok(".git/description") then if ok(".git/description") then
append_short_version("Repository description", replies[".git/description"].body) loc["repository-description"] = replies[".git/description"].body
end end
-- If we got /.git/config, we might find out things like the user's GitHub name, -- .git/config contains a list of remotes, so we try to extract them.
-- if they have a Heroku remote, whether this is a bare repository or not (if it
-- is bare, that means it's likely a remote for other people), and in future
-- versions of Git when there are more than one repo format version, we will
-- display that too.
if ok(".git/config") then if ok(".git/config") then
local config = replies[".git/config"].body local config = replies[".git/config"].body
local remotes = {} local remotes = {}
@@ -197,36 +176,21 @@ function action(host, port)
table.insert(remotes, url) table.insert(remotes, url)
end end
-- These are some popular / well-known Git hosting services and/or hosting services
-- that allow deployment via 'git push'
local popular_remotes = {
["github.com"] = "Source might be at https://github.com/<reponame>",
["gitorious.com"] = "Source might be at https://gitorious.com/<reponame>",
["bitbucket.org"] = "Source might be at https://bitbucket.org/<reponame>",
["heroku.com"] = "App might be deployed to http://<reponame>.herokuapp.com",
}
for _, url in ipairs(remotes) do for _, url in ipairs(remotes) do
out:insert("Remote: " .. url) loc["remotes"] = loc["remotes"] or {}
local domain, reponame = string.match(url, "[@/]([%w._-]+)[:/]([%w._-]+/?[%w._-]+)") table.insert(loc["remotes"], url)
local extrainfo = popular_remotes[domain]
-- Try and cut off the '.git' extension
reponame = string.match(reponame, "(.+)%.git") or reponame
if extrainfo then
out:insert(" -> " .. string.gsub(extrainfo, "<reponame>", reponame))
end
end end
end end
-- These are files that are used by Git to determine -- These are files that are used by Git to determine what files to ignore.
-- what files to ignore. We use this list to make the -- We use this list to make the loop below (used to determine what kind of
-- loop below (used to determine what kind of application -- application is in the repository) more generic.
-- is in the repository) more generic
local ignorefiles = { local ignorefiles = {
".gitignore", ".gitignore",
".git/info/exclude", ".git/info/exclude",
} }
local fingerprints = { local fingerprints = {
-- Many of these taken from https://github.com/gitignore -- Many of these taken from https://github.com/github/gitignore
{ "%.scala_dependencies", "Scala application" }, { "%.scala_dependencies", "Scala application" },
{ "npm%-debug%.log", "node.js application" }, { "npm%-debug%.log", "node.js application" },
{ "joomla%.xml", "Joomla! site" }, { "joomla%.xml", "Joomla! site" },
@@ -242,16 +206,17 @@ function action(host, port)
{ "%.class", "Java application" }, { "%.class", "Java application" },
{ "%.php", "PHP application" }, { "%.php", "PHP application" },
} }
local excludefile_that_matched = nil -- The XML produced here is divided by ignorefile and is sorted from first to last
local app = nil -- in order of specificity. e.g. All JBoss applications are Java applications,
-- We check every file against every fingerprint -- but not all Java applications are JBoss. In that case, JBoss and Java will
-- be output, but JBoss will be listed first.
for _, file in ipairs(ignorefiles) do for _, file in ipairs(ignorefiles) do
if ok(file) then -- we only test all fingerprints if we got the file if ok(file) then -- We only test all fingerprints if we got the file.
for i, fingerprint in ipairs(fingerprints) do for _, fingerprint in ipairs(fingerprints) do
if string.match(replies[file].body, fingerprint[1]) then if string.match(replies[file].body, fingerprint[1]) then
out:insert("Based on the file '" .. file .. "', this is a " .. fingerprint[2]) loc["project-type"] = loc["project-type"] or {}
-- Based on the file '.gitignore', this is a Ruby on Rails application" loc["project-type"][file] = loc["project-type"][file] or {}
break -- we only want to print our first guess (the most specific one that matched) table.insert(loc["project-type"][file], fingerprint[2])
end end
end end
end end
@@ -259,7 +224,86 @@ function action(host, port)
end end
end end
-- Replace non-printing characters with asterisks -- If we didn't get anything, we return early. No point doing the
if #out > 0 then return stdnse.format_output(true, out) -- normal formatting!
else return nil end if out == nil then
return nil
end
-- Truncate to TRUNC_LENGTH characters and replace control characters (newlines, etc) with spaces.
local function summarize(str)
str = stdnse.string_or_blank(str, "<unknown>")
local original_length = #str
str = string.sub(str, 1, TRUNC_LENGTH)
str = string.gsub(str, "%c", " ")
if original_length > TRUNC_LENGTH then
str = str .. "..."
end
return str
end
-- We convert the full output to pretty output for -oN
local normalout
for location, info in pairs(out) do
normalout = normalout or {}
-- This table gets converted to a string format_output, and then inserted into the 'normalout' table
local new = {}
-- Headings for each place we found a repo
new["name"] = location
-- How sure are we that this is a Git repository?
local count = { tried = 0, ok = 0 }
for _, found in pairs(info["files-found"]) do
count.tried = count.tried + 1
if found then count.ok = count.ok + 1 end
end
-- If 3 or more of the files we were looking for are not on the server,
-- we are less confident that we got a real Git repository
if count.tried - count.ok <= 2 then
table.insert(new, "Git repository found!")
else -- We already got .git/HEAD, so we add 1 to 'tried' and 'ok'
table.insert(new, "Potential Git repository found (found " .. (count.ok + 1) .. "/" .. (count.tried + 1) .. " expected files)")
end
-- Show what patterns matched what files
for name, matches in pairs(info["interesting-matches"] or {}) do
local temp = name .. " matched patterns"
for _, matched in ipairs(matches) do
temp = temp .. " '" .. matched .. "'"
end
table.insert(new, temp)
end
if info["repository-description"] then
table.insert(new, "Repository description: " .. summarize(info["repository-description"]))
end
if info["last-commit-message"] then
table.insert(new, "Last commit message: " .. summarize(info["last-commit-message"]))
end
-- If we found any remotes in .git/config, process them now
if info["remotes"] then
local old_name = info["remotes"]["name"] -- in case 'name' is a remote
info["remotes"]["name"] = "Remotes:"
-- Remove the newline from format_output's output - it looks funny with it
local temp = string.gsub(stdnse.format_output(true, info["remotes"]), "^\n", "")
-- using 'temp' here because gsub() has multiple return values that insert() will try
-- to use, and I don't know of a better way to prevent that ;)
table.insert(new, temp)
info["remotes"]["name"] = old_name
end
-- Take the first guessed project type from each ignorefile
if info["project-type"] then
for name, types in pairs(info["project-type"]) do
table.insert(new, "Project type: " .. types[1] .. " (guessed from " .. name .. ")")
end
end
-- Insert this location's information.
table.insert(normalout, new)
end
return out, stdnse.format_output(true, normalout)
end end