Revision as of 06:25, 20 November 2020

The following SQL queries were found to be useful when developing/debugging Truxton.

Database Server Information

Number of Connections

SELECT COUNT(DISTINCT(numbackends)) FROM pg_stat_database

Maximum Number of Connections

Run this query as an administrator.

SHOW max_connections

All Connection Information

This query will show you the number of active connections, maximum number of connections (server configuration) and the number of connections reserved for the super user.

SELECT * FROM
(SELECT COUNT(*) "Current Number of Connections" FROM pg_stat_activity) q1,
(SELECT setting::int "Reserved for Superuser" FROM pg_settings WHERE name=$$superuser_reserved_connections$$) q2,
(SELECT setting::int "Maximum Number of Connections" FROM pg_settings WHERE name=$$max_connections$$) q3;

Current Activity

Ever wonder what the server is doing? This will show you what each connection is doing.

SELECT state_change,
       age(localtimestamp, state_change),
       datname,
       client_addr,
       application_name,
       state,
       query
FROM pg_stat_activity 
ORDER BY state,
         age DESC,
         datname ASC

Terminate Connection

PostgreSQL 12 will sometimes leave zombie connections open for days. Here's how to kill them. Each connection is a different process on the server.

Kill Connection By Process ID

SELECT pg_terminate_backend(pid) 
FROM pg_stat_activity
WHERE pid = '29564'

Kill Connection By Application Name

Truxton processes will identify themselves via the application name, making it easier to kill them.

SELECT pg_terminate_backend(pid) 
FROM pg_stat_activity
WHERE application_name like 'Truxton Carve%'

Files

Queries to get file lists.

Human Readable List

This will list files in a human readable format. The WHERE clause is commented out so you can add your own filter criteria.

SELECT "File"."ID",
       "File"."MediaID",
       "File"."ParentFileID",
       "Origin"."Name" AS "Origin",
       "FileType"."ShortName" AS "FileType",
       "FN2"."Name",
       "File"."OSLength",
       "File"."HashID",
       "File"."RawEntropy" * 0.00024414502395702993865779595324564 AS "Entropy",
       "Filename"."Name" AS "Path",
       "File"."FullPathID",
       "File"."FileTypeID",
       to_hex( "File"."Signature" ) AS "Signature",
       to_hex( "File"."PhysicalDiskOffset" ) AS "DiskOffset",
       "FileType"."LongName" AS "FileTypeDescription",
       "File"."Created",
       "File"."LastWrite",
       "ContentStatus"."Name" AS "CStatus",
       "File"."NumberOfChildren",
       "File"."FilenameID"
FROM "File"
LEFT OUTER JOIN "FileType" ON ( "File"."FileTypeID" = "FileType"."ID" )
LEFT OUTER JOIN "Filename" AS "FN2" ON ( "File"."FilenameID" = "FN2"."ID" )
LEFT OUTER JOIN "Filename" ON ( "File"."FullPathID" = "Filename"."ID" )
LEFT OUTER JOIN "Origin" ON ( "File"."OriginID" = "Origin"."ID" )
LEFT OUTER JOIN "ContentStatus" ON ( "File"."ContentStatusID" = "ContentStatus"."ID" )
--WHERE "File"."Signature" = -623191334
ORDER BY "File"."ID"

Files with Depot Locations

SELECT "File"."ID",
       "File"."ParentFileID",
       "File"."FileTypeID",
       "FileType"."ShortName",
       "Filename"."Name",
       "Content"."Offset",
       "Content"."Length",
       "Depot"."Filename"
FROM "File", 
     "Depot",
     "Filename",
     "Content",
     "FileType"
WHERE "File"."HashID" = "Content"."Hash" 
  AND "File"."FilenameID" = "Filename"."ID"
  AND "Content"."DepotID" = "Depot"."ID"
  AND "File"."FileTypeID" = "FileType"."ID"

Files Truxton Can Exploit

SELECT "ShortName",
       "LongName"
FROM "FileType"
WHERE "ID" IN
(
  SELECT DISTINCT("FileTypeID")
  FROM "ETLRoute"
  WHERE "ETLQueueName" IN ('expand', 'email', 'remoteexpand', 'archives', 'pfe                             ')
)
ORDER BY "ShortName"

This one produces a single line of output that is easier for a human to read"

SELECT CONCAT( "ShortName", ' - ', "LongName" )
FROM "FileType"
WHERE "ID" IN
(
  SELECT DISTINCT("FileTypeID")
  FROM "ETLRoute"
  WHERE "ETLQueueName" IN ('expand', 'email', 'remoteexpand', 'archives', 'pfe                             ')
)
ORDER BY "ShortName"

NOTE the pfe item is a bug. There should NOT be spaces in that name.

Logs

All Messages

SELECT "Log"."MediaID",
       "Log"."When",
       "T"."String" AS "Type",
       "S"."String" AS "Source",
       "Log"."ProcessID",
       "Log"."ThreadID",
       "C"."String" AS "Client",
       "U"."String" AS "User",
       "M"."String" AS "Message"
FROM "Log",
     "LogString" AS "S",
     "LogString" AS "C",
     "LogString" AS "U",
     "LogString" AS "M",
     "LogString" AS "T"
WHERE "Log"."LogStringSourceID" = "S"."ID" 
  AND "Log"."LogStringClientID" = "C"."ID" 
  AND "Log"."LogStringUserID" = "U"."ID" 
  AND "Log"."LogStringMessageID" = "M"."ID" 
  AND "Log"."LogStringTypeID" = "T"."ID"
-- AND "Log"."LogStringTypeID" <> 2 -- Don't show Info messages
-- AND "Log"."MediaID" = '11111111-1111-1111-1111-000000000001'::uuid
ORDER BY "Log"."MediaID",
         "Log"."When"

Only Error and Warnings

SELECT "Log"."MediaID",
       "Log"."When",
       "Log"."LogStringTypeID",
       "T"."String" AS "Type",
       "S"."String" AS "Source",
       "Log"."ProcessID",
       "Log"."ThreadID",
       "C"."String" AS "Client",
       "U"."String" AS "User",
       "M"."String" AS "Message"
FROM "Log",
     "LogString" AS "S",
     "LogString" AS "C",
     "LogString" AS "U",
     "LogString" AS "M",
     "LogString" AS "T"
WHERE "Log"."LogStringSourceID" = "S"."ID" 
  AND "Log"."LogStringClientID" = "C"."ID" 
  AND "Log"."LogStringUserID" = "U"."ID" 
  AND "Log"."LogStringMessageID" = "M"."ID" 
  AND "Log"."LogStringTypeID" = "T"."ID"
  AND "Log"."LogStringTypeID" IN ( 0, 1 ) -- Don't show Info messages
-- AND "Log"."MediaID" = '11111111-1111-1111-1111-000000000001'::uuid
ORDER BY "Log"."MediaID",
         "Log"."When"

Load Log

The following query is used to generate the Load Log. You can run this query as media is being loaded to peek at the log.

SELECT "Log"."When",
       "T"."String" AS "Type",
       "S"."String" AS "Source",
       "Log"."ProcessID",
       "Log"."ThreadID",
       "C"."String" AS "Client",
       "U"."String" AS "User",
       "M"."String" AS "Message",
       "Log"."LogStringMessageID"
FROM "Log",
     "LogString" AS "S",
     "LogString" AS "C",
     "LogString" AS "U",
     "LogString" AS "M",
     "LogString" AS "T"
WHERE "Log"."MediaID" = '11111111-1111-1111-1111-000000000001'::uuid
  AND "Log"."LogStringSourceID" = "S"."ID"
  AND "Log"."LogStringClientID" = "C"."ID"
  AND "Log"."LogStringUserID" = "U"."ID"
  AND "Log"."LogStringMessageID" = "M"."ID"
  AND "Log"."LogStringTypeID" = "T"."ID"
ORDER BY "Log"."When",
         "Source";

Depots

These queries will tell you if you're missing depot file entries.

Missing Content Depots

SELECT DISTINCT( "DepotID" )
FROM "Content"
WHERE "DepotID" <> '00000000-0000-0000-0000-000000000000'::uuid
  AND "DepotID" NOT IN ( SELECT "ID" FROM "Depot" )
ORDER BY "DepotID"

Missing Image Depots

SELECT DISTINCT( "DepotID" )
FROM "Image"
WHERE "DepotID" <> '00000000-0000-0000-0000-000000000000'::uuid
  AND "DepotID" NOT IN
  (
    SELECT "ID" FROM "Depot"
  )
ORDER BY "DepotID"

Depot IDs in Use

SELECT DISTINCT "DepotID" AS "ID" FROM
(
  SELECT "DepotID" FROM "Content"
  UNION
  SELECT "DepotID" FROM "Free"
  UNION
  SELECT "DepotID" FROM "Image"
  UNION
  SELECT "DepotID" FROM "Slack"
  UNION
  SELECT "ThumbnailDepotID" AS "DepotID" FROM "ReviewItem"
) AS "SubSelect"

Depot IDs Not in Use

SELECT "ID"
FROM "Depot"
WHERE "ID" NOT IN
(
 SELECT DISTINCT "DepotID" AS "ID" FROM
 (
  SELECT "DepotID" FROM "Content"
  UNION
  SELECT "DepotID" FROM "Free"
  UNION
  SELECT "DepotID" FROM "Image"
  UNION
  SELECT "DepotID" FROM "Slack"
  UNION
  SELECT "ThumbnailDepotID" AS "DepotID" FROM "ReviewItem"
 ) AS "SubSelect"
)

URI in Filename Column

To detect when an ETL has put bad data into the [Depot] table.

SELECT "ID",
       "Filename",
       "URI",
       "DepotTypeID",
       "MaximumSize",
       "Size",
       "IsLongTerm",
       "CryptoKey",
       "DepotStatusID"
FROM "Depot"
WHERE "Filename" LIKE '\\%'

Change Depot URI

When you move depots to a new share, the database needs to be fixed up. This sample is what you would run if you need to change the machine name SOFS-01 to a fully qualified DNS name.

UPDATE "Depot"
SET "URI" = REPLACE( "URI", '\\SOFS-01\', '\\sofs-01.truxton.lab\Depot\')

Change Depot Path

If you move depots to a new folder, you must update the [Filename] column of the [Depot] table. This sample is what you would do if you moved the depot folder from C:\TruxtonData to C:\Storage

UPDATE "Depot"
SET "Filename" = REPLACE( "Filename", 'C:\Truxton Data\Depot\', 'C:\Storage\Depot\')

File Type Counts

SELECT "FileTypeID",
       "FileType"."ShortName",
       COUNT( "FileTypeID" ) AS "NumberOfThatType"
FROM "File"
INNER JOIN "FileType" ON ( "FileType"."ID" = "File"."FileTypeID" )
GROUP BY "FileTypeID",
         "FileType"."ShortName"
ORDER BY "NumberOfThatType" DESC

Messages

Messages and Subjects

SELECT "Message"."ID",
       "MessageType"."ShortName" AS "Type",
       "Sent", "Received", 
       "MessageSubject"."Text" AS "Subject",
       "NumberOfBodies",
       "NumberOfAttachments"
FROM "Message"
LEFT OUTER JOIN "MessageSubject" ON ( "Message"."MessageSubjectID" = "MessageSubject"."ID" )
LEFT OUTER JOIN "MessageType" ON ("Message"."MessageTypeID" = "MessageType"."ID" )
ORDER BY "Message"."Sent"

Find Messages that did not Parse

SELECT "File"."ID",
       "FN2"."Name",
       "File"."OSLength",
       "File"."HashID",
       "ContentStatus"."Name" AS "CStatus",
       "File"."NumberOfChildren"
FROM "File"
LEFT OUTER JOIN "Filename" AS "FN2" ON ( "File"."FilenameID" = "FN2"."ID" )
LEFT OUTER JOIN "Origin" ON ( "File"."OriginID" = "Origin"."ID" )
LEFT OUTER JOIN "ContentStatus" ON ( "File"."ContentStatusID" = "ContentStatus"."ID" )
WHERE "File"."FileTypeID" = 199
  AND "File"."ID" NOT IN
  (
    SELECT "FileID" FROM "Message
  )

SSID

SSID and Associated Passwords

SELECT DISTINCT "A_ID" AS "PasswordEntityID",
       "B_ID" AS "SSIDEntityID",
       "SSID"."Value" AS "SSID",
       "Password"."Value" AS "Password"
FROM "Relation"
INNER JOIN "Entity" AS "PasswordEntity" ON ( "Relation"."A_ID" = "PasswordEntity"."ID" )
INNER JOIN "Entity" AS "SSIDEntity" ON ( "Relation"."B_ID" = "SSIDEntity"."ID" )
INNER JOIN "EntityString" AS "Password" ON ("PasswordEntity"."EntityStringID" = "Password"."ID")
INNER JOIN "EntityString" AS "SSID" ON ("SSIDEntity"."EntityStringID" = "SSID"."ID")
WHERE "B_ID" IN 
(
  SELECT "ID"
  FROM "Entity"
  WHERE "MediaID" = '0e8beeee-1c42-91c3-e0b1-93edf6ed2d73'::uuid
    AND "EntityTypeID" = 25
)
ORDER BY "SSID"."Value"

Entity

Human Readable

How to get the entities to reprocess a piece of media.

SELECT "Entity"."ID",
       "Entity"."FileID", 
       "Entity"."EntityTypeID", 
       "Entity"."EntityStringID",
       "Entity"."Offset",
       "Entity"."ObjectID", 
       "Entity"."ObjectTypeID",
       "Entity"."Length",
       "Entity"."DataTypeID",
       "ObjectType"."Name", 
       "EntityString"."Value", 
       "EntityType"."LongName"
FROM "Entity"
INNER JOIN "EntityType" ON ( "Entity"."EntityTypeID" = "EntityType"."ID" )
INNER JOIN "EntityString" ON ( "Entity"."EntityStringID" = "EntityString"."ID" )
INNER JOIN "ObjectType" ON ( "Entity"."ObjectTypeID" = "ObjectType"."ID" )
WHERE "Entity"."MediaID" = '139db997-1f71-58d2-46ae-e052bb67a946'::uuid

Loads

How Long did the Loads Take

This query is useful only when you are loading a bunch of media for an investigation at the same time.

SELECT MIN("Start"),
       MAX("End")
FROM "Event"
WHERE "EventTypeID" = 1002
  AND "MediaID" IN
  (
    SELECT "MediaID" FROM "InvestigationMedia" WHERE "InvestigationID" = '20171021-0000-1111-2222-333333333333'::uuid
  )

Links

The following links have proven useful:

Difference between revisions of "Useful SQL Queries"