PenTest/nmap
1
0
Fork 0
mirror of https://github.com/nmap/nmap.git synced 2025-12-06 04:31:29 +00:00
Code Issues Projects Releases Wiki Activity

Add structured output to http-get.nse.

Browse Source 
This patch is by Alex Weber.
http://seclists.org/nmap-dev/2012/q4/434
This commit is contained in:
david
2012-12-13 04:16:01 +00:00
parent a8ed9f57f8
commit 43f28b8f4a
2 changed files with 182 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

316
scripts/http-git.nse
View File

@@ -1,43 +1,70 @@
-- Checks for a Git repository found in a website's document root (GET /.git/<something> HTTP/1.1)
-- Gets as much information about the repository as possible, including language/framework, Github
-- username, last commit message, and repository description.
--
-- @output
-- PORT STATE SERVICE
-- 80/tcp open http
-- | http-git:
-- | Git repository found in web root
-- | Last commit message: This is my last commit...
-- | Repository description: Unnamed repository; edit this file 'description' to name the...
-- | GitHub remote: AlexWebr/nse (accessed over SSH)
-- | BitBucket remote: AlexWebr/nse (accessed over HTTP, pull-only)
-- |_ Based on the file '.gitignore', this is a Ruby on Rails application
--
-- Version 1.1
-- Created 27 June 2012 - written by Alex Weber <alexwebr@gmail.com>
local http = require("http")
local shortport = require("shortport")
local stdnse = require("stdnse")
local strbuf = require("strbuf")
local string = require("string")
local table = require("table")
description = [[ Checks for a Git repository found in a website's document root (/.git/<something>) then retrieves as much repo information as possible, including language/framework, Github username, last commit message, and repository description.
description = [[
Checks for a Git repository found in a website's document root
/.git/<something>) and retrieves as much repo information as
possible, including language/framework, remotes, last commit
message, and repository description.
]]
categories = { "safe", "vuln", "default" }
-- @output
-- PORT STATE SERVICE REASON
-- 80/tcp open http syn-ack
-- | http-git:
-- | 127.0.0.1:80/.git/
-- | Git repository found!
-- | .git/config matched patterns 'passw'
-- | Repository description: Unnamed repository; edit this file 'description' to name the...
-- | Remotes:
-- | http://github.com/someuser/somerepo
-- | Project type: Ruby on Rails web application (guessed from .git/info/exclude)
-- | 127.0.0.1:80/damagedrepository/.git/
-- |_ Potential Git repository found (found 2/6 expected files)
--
-- @xmloutput
-- <table key="127.0.0.1:80/.git/">
-- <table key="remotes">
-- <elem>http://github.com/anotherperson/anotherepo</elem>
-- </table>
-- <table key="project-type">
-- <table key=".git/info/exclude">
-- <elem>JBoss Java web application</elem>
-- <elem>Java application</elem>
-- </table>
-- </table>
-- <elem key="repository-description">A nice repository</elem>
-- <table key="files-found">
-- <elem key=".git/COMMIT_EDITMSG">false</elem>
-- <elem key=".git/info/exclude">true</elem>
-- <elem key=".git/config">true</elem>
-- <elem key=".git/description">true</elem>
-- <elem key=".gitignore">false</elem>
-- </table>
-- <table key="interesting-matches">
-- <table key=".git/config">
-- <elem>passw</elem>
-- </table>
-- </table>
-- </table>
categories = { "default", "safe", "vuln" }
author = "Alex Weber"
license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
portrule = shortport.http
local STATUS_OK = 200 -- We consider 200 to mean "okay, file exists and we received its contents"
-- We consider 200 to mean "okay, file exists and we received its contents".
local STATUS_OK = 200
-- Long strings (like a repository's description) will be truncated to this
-- number of characters in normal output.
local TRUNC_LENGTH = 60
function action(host, port)
-- All methods that we call on this table will be from the table library
local out = {}
setmetatable(out, {__index = table})
local repos_found = 0
local out
-- We can accept a single root, or a table of roots to try
local root_arg = stdnse.get_script_args("http-git.root")
@@ -51,7 +78,7 @@ function action(host, port)
end
-- Try each root in succession
for i, root in ipairs(roots) do
for _, root in ipairs(roots) do
root = tostring(root)
root = root or '/'
@@ -68,10 +95,7 @@ function action(host, port)
-- We could try for /.git/, but we will not get a 200 if directory
-- listings are disallowed.
if http.get(host, port, root .. ".git/HEAD").status == STATUS_OK then
if repos_found > 0 then
out:insert("")
end
repos_found = repos_found + 1
out = out or {}
local replies = {}
-- This function returns true if we got a 200 OK when
-- fetching 'filename' from the server
@@ -83,111 +107,66 @@ function action(host, port)
-- These files are created by creating and using the repository,
-- or by popular development frameworks.
local repo = {
".gitignore",
".git/COMMIT_EDITMSG",
".git/config",
".git/description",
".git/info/exclude",
".git/COMMIT_EDITMSG",
".gitignore",
}
local count = { ok = 0, tried = 0 }
local prequests = {} -- prequests = pipelined requests (temp)
local pl_requests = {} -- pl_requests = pipelined requests (temp)
-- Go through all of the filenames and do an HTTP GET
for _, name in ipairs(repo) do -- for every filename
http.pipeline_add(root .. name, nil, prequests)
http.pipeline_add(root .. name, nil, pl_requests)
end
-- do the requests
replies = http.pipeline_go(host, port, prequests)
-- Do the requests.
replies = http.pipeline_go(host, port, pl_requests)
if replies == nil then
stdnse.print_debug("%s: pipeline_go() error. Aborting.", SCRIPT_NAME)
return nil
end
for i, reply in ipairs(replies) do
count.tried = count.tried + 1
-- We want this to be indexed by filename, not an integer, so we convert it
-- We added to the pipeline in the same order as the filenames, so this is safe
-- We added to the pipeline in the same order as the filenames, so this is safe.
replies[repo[i]] = reply -- create index by filename
replies[i] = nil -- delete integer-indexed entry
if reply.status == STATUS_OK then count.ok = count.ok + 1 end
end
-- Tell the user that we found a repository, and indicate if
-- we didn't find all the files we were looking for.
-- Mark each file that we tried to get as 'found' (true) or 'not found' (false).
local location = host.ip .. ":" .. port.number .. root .. ".git/"
if count.ok == count.tried then
out:insert("Git repository found at " .. location)
else -- if we didn't find all the files we were hoping to, we might not actually have a repo
out:insert("Potential Git repository found at " .. location .. " (found " ..
tostring(count.ok + 1) .. " of " .. tostring(count.tried + 1) .. " expected files)")
-- we already got /.git/HEAD, so add one to 'found' and 'expected'
out[location] = {}
-- A nice shortcut
local loc = out[location]
loc["files-found"] = {}
for name, _ in pairs(replies) do
loc["files-found"][name] = ok(name)
end
-- This function matches a table of words/regexes against a single string
-- This function is used immediately after it is declared
local function match_many(str, table_of_words)
local matched_string, lstr, t_to_return = false, string.lower(str), {}
for i, word in ipairs(table_of_words) do
matched_string = string.match(lstr, word)
if matched_string then table.insert(t_to_return , matched_string) end
end
return t_to_return
end
-- Look through all the repo files we grabbed and see if we can find anything interesting
local interesting = { "bug", "passw", "pw", "user", "uid", "key", "secret" }
for name, reply in pairs(replies) do
if ok(name) then -- for all replies that were successful
local found_anything = false -- have we found anything yet?
local buf = strbuf.new()
for _, matched in ipairs(match_many(reply.body:lower(), interesting)) do -- match all files against 'interesting'
if not found_anything then -- if this is our first find, print filename and stuff
buf = (((((buf .. "Contents of '") .. name) .. "' matched patterns '") .. matched) .. "'") -- the '..' is right-associative :(
found_anything = true
else
buf = ((buf .. ", '" .. matched) .. "'")
end -- if we found something already, tack this pattern onto the end
end -- If we matched anything, finish the line up
if found_anything then
buf = buf .. " (case-insensitive match)"
out:insert(strbuf.dump(buf))
-- Look through all the repo files we grabbed and see if we can find anything interesting.
local interesting = { "bug", "key", "passw", "pw", "user", "secret", "uid" }
for name, reply in pairs(replies) do
if ok(name) then
for _, pattern in ipairs(interesting) do
if string.match(reply.body, pattern) then
-- A Lua idiom - don't create this table until we actually have something to put in it
loc["interesting-matches"] = loc["interesting-matches"] or {}
loc["interesting-matches"][name] = loc["interesting-matches"][name] or {}
table.insert(loc["interesting-matches"][name], pattern)
end
end
end
end
-- Given a longer plain-text string (a large text file, for example), append
-- a summary of it (the first 60 characters or the first line, whichever is shorter)
local function append_short_version(description, original_string)
local short = string.sub(original_string, 1, 60) -- trim the string first, in case it is huge
-- We try to cut off the newline if we can
local no_newline = string.match(short, "(.-)\r?\n") -- we don't want such an open-ended regex on a potentially huge string
local s = no_newline or short
if #original_string > #s then
-- If we cut off something, we want to put an ellipsis on the end
s = description .. ": " .. s .. "..."
else
s = description .. ": " .. s
end
out:insert(s)
end
-- Get the first line and trim to 60 characters, if we got a COMMIT_EDITMESSAGE
if ok(".git/COMMIT_EDITMSG") then
-- If there's no newline in the file (there usually is), this won't work.
-- Normally, it's a commit message followed by a bunch of commented-out lines (#).
append_short_version("Last commit message", replies[".git/COMMIT_EDITMSG"].body)
loc["last-commit-message"] = replies[".git/COMMIT_EDITMSG"].body
end
-- Do the same thing as for description - get first line, truncate to 60 characters.
if ok(".git/description") then
append_short_version("Repository description", replies[".git/description"].body)
loc["repository-description"] = replies[".git/description"].body
end
-- If we got /.git/config, we might find out things like the user's GitHub name,
-- if they have a Heroku remote, whether this is a bare repository or not (if it
-- is bare, that means it's likely a remote for other people), and in future
-- versions of Git when there are more than one repo format version, we will
-- display that too.
-- .git/config contains a list of remotes, so we try to extract them.
if ok(".git/config") then
local config = replies[".git/config"].body
local remotes = {}
@@ -197,36 +176,21 @@ function action(host, port)
table.insert(remotes, url)
end
-- These are some popular / well-known Git hosting services and/or hosting services
-- that allow deployment via 'git push'
local popular_remotes = {
["github.com"] = "Source might be at https://github.com/<reponame>",
["gitorious.com"] = "Source might be at https://gitorious.com/<reponame>",
["bitbucket.org"] = "Source might be at https://bitbucket.org/<reponame>",
["heroku.com"] = "App might be deployed to http://<reponame>.herokuapp.com",
}
for _, url in ipairs(remotes) do
out:insert("Remote: " .. url)
local domain, reponame = string.match(url, "[@/]([%w._-]+)[:/]([%w._-]+/?[%w._-]+)")
local extrainfo = popular_remotes[domain]
-- Try and cut off the '.git' extension
reponame = string.match(reponame, "(.+)%.git") or reponame
if extrainfo then
out:insert(" -> " .. string.gsub(extrainfo, "<reponame>", reponame))
end
loc["remotes"] = loc["remotes"] or {}
table.insert(loc["remotes"], url)
end
end
-- These are files that are used by Git to determine
-- what files to ignore. We use this list to make the
-- loop below (used to determine what kind of application
-- is in the repository) more generic
-- These are files that are used by Git to determine what files to ignore.
-- We use this list to make the loop below (used to determine what kind of
-- application is in the repository) more generic.
local ignorefiles = {
".gitignore",
".git/info/exclude",
}
local fingerprints = {
-- Many of these taken from https://github.com/gitignore
-- Many of these taken from https://github.com/github/gitignore
{ "%.scala_dependencies", "Scala application" },
{ "npm%-debug%.log", "node.js application" },
{ "joomla%.xml", "Joomla! site" },
@@ -242,16 +206,17 @@ function action(host, port)
{ "%.class", "Java application" },
{ "%.php", "PHP application" },
}
local excludefile_that_matched = nil
local app = nil
-- We check every file against every fingerprint
-- The XML produced here is divided by ignorefile and is sorted from first to last
-- in order of specificity. e.g. All JBoss applications are Java applications,
-- but not all Java applications are JBoss. In that case, JBoss and Java will
-- be output, but JBoss will be listed first.
for _, file in ipairs(ignorefiles) do
if ok(file) then -- we only test all fingerprints if we got the file
for i, fingerprint in ipairs(fingerprints) do
if ok(file) then -- We only test all fingerprints if we got the file.
for _, fingerprint in ipairs(fingerprints) do
if string.match(replies[file].body, fingerprint[1]) then
out:insert("Based on the file '" .. file .. "', this is a " .. fingerprint[2])
-- Based on the file '.gitignore', this is a Ruby on Rails application"
break -- we only want to print our first guess (the most specific one that matched)
loc["project-type"] = loc["project-type"] or {}
loc["project-type"][file] = loc["project-type"][file] or {}
table.insert(loc["project-type"][file], fingerprint[2])
end
end
end
@@ -259,7 +224,86 @@ function action(host, port)
end
end
-- Replace non-printing characters with asterisks
if #out > 0 then return stdnse.format_output(true, out)
else return nil end
-- If we didn't get anything, we return early. No point doing the
-- normal formatting!
if out == nil then
return nil
end
-- Truncate to TRUNC_LENGTH characters and replace control characters (newlines, etc) with spaces.
local function summarize(str)
str = stdnse.string_or_blank(str, "<unknown>")
local original_length = #str
str = string.sub(str, 1, TRUNC_LENGTH)
str = string.gsub(str, "%c", " ")
if original_length > TRUNC_LENGTH then
str = str .. "..."
end
return str
end
-- We convert the full output to pretty output for -oN
local normalout
for location, info in pairs(out) do
normalout = normalout or {}
-- This table gets converted to a string format_output, and then inserted into the 'normalout' table
local new = {}
-- Headings for each place we found a repo
new["name"] = location
-- How sure are we that this is a Git repository?
local count = { tried = 0, ok = 0 }
for _, found in pairs(info["files-found"]) do
count.tried = count.tried + 1
if found then count.ok = count.ok + 1 end
end
-- If 3 or more of the files we were looking for are not on the server,
-- we are less confident that we got a real Git repository
if count.tried - count.ok <= 2 then
table.insert(new, "Git repository found!")
else -- We already got .git/HEAD, so we add 1 to 'tried' and 'ok'
table.insert(new, "Potential Git repository found (found " .. (count.ok + 1) .. "/" .. (count.tried + 1) .. " expected files)")
end
-- Show what patterns matched what files
for name, matches in pairs(info["interesting-matches"] or {}) do
local temp = name .. " matched patterns"
for _, matched in ipairs(matches) do
temp = temp .. " '" .. matched .. "'"
end
table.insert(new, temp)
end
if info["repository-description"] then
table.insert(new, "Repository description: " .. summarize(info["repository-description"]))
end
if info["last-commit-message"] then
table.insert(new, "Last commit message: " .. summarize(info["last-commit-message"]))
end
-- If we found any remotes in .git/config, process them now
if info["remotes"] then
local old_name = info["remotes"]["name"] -- in case 'name' is a remote
info["remotes"]["name"] = "Remotes:"
-- Remove the newline from format_output's output - it looks funny with it
local temp = string.gsub(stdnse.format_output(true, info["remotes"]), "^\n", "")
-- using 'temp' here because gsub() has multiple return values that insert() will try
-- to use, and I don't know of a better way to prevent that ;)
table.insert(new, temp)
info["remotes"]["name"] = old_name
end
-- Take the first guessed project type from each ignorefile
if info["project-type"] then
for name, types in pairs(info["project-type"]) do
table.insert(new, "Project type: " .. types[1] .. " (guessed from " .. name .. ")")
end
end
-- Insert this location's information.
table.insert(normalout, new)
end
return out, stdnse.format_output(true, normalout)
end