Traccar v6.14.4 failed to decode coordinates while CLOSE-WAIT large
Are you sure it's 6.14.4 and not 6.14.3?
yes, its 6.14.4
# grep -ri "Version:" /opt/traccar/logs/
grep: /opt/traccar/logs/tracker-server.log: binary file matches
/opt/traccar/logs/tracker-server.log.20260605:2026-06-05 17:44:55 INFO: Operating system name: Linux version: 5.15.0-97-generic architecture: amd64
/opt/traccar/logs/tracker-server.log.20260605:2026-06-05 17:44:55 INFO: Java runtime name: OpenJDK 64-Bit Server VM vendor: Eclipse Adoptium version: 25.0.3+9-LTS
/opt/traccar/logs/tracker-server.log.20260605:2026-06-05 17:44:55 INFO: Version: 6.14.4
/opt/traccar/logs/tracker-server.log.20260605:2026-06-05 17:45:14 DEBUG: Java version: 25
I recommend checking database queue.
I found out the issue is at CacheManager.java at addDevice and removeDevice portion.
When many device goes in after restart, it block the process.
After i modify the function and rebuild the tracker-server.jar the decode back to normal now.
What was the modification?
Can you share your custom function for CacheManager.java ?
Hi,
I just move some process outside the netty lock and create pre-warm cache.
Below are my CacheManager.java
/*
* Copyright 2022 - 2026 Anton Tananaev (anton@traccar.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.traccar.session.cache;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.traccar.broadcast.BroadcastInterface;
import org.traccar.broadcast.BroadcastService;
import org.traccar.config.Config;
import org.traccar.config.Keys;
import org.traccar.helper.model.AttributeUtil;
import org.traccar.helper.model.PositionUtil;
import org.traccar.model.Attribute;
import org.traccar.model.BaseModel;
import org.traccar.model.Calendar;
import org.traccar.model.Device;
import org.traccar.model.Driver;
import org.traccar.model.Geofence;
import org.traccar.model.Group;
import org.traccar.model.GroupedModel;
import org.traccar.model.LinkedDevice;
import org.traccar.model.Maintenance;
import org.traccar.model.Notification;
import org.traccar.model.ObjectOperation;
import org.traccar.model.Permission;
import org.traccar.model.Position;
import org.traccar.model.Schedulable;
import org.traccar.model.Server;
import org.traccar.model.User;
import org.traccar.storage.Storage;
import org.traccar.storage.StorageException;
import org.traccar.storage.query.Columns;
import org.traccar.storage.query.Condition;
import org.traccar.storage.query.Request;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Deque;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@Singleton
public class CacheManager implements BroadcastInterface {
private static final Logger LOGGER = LoggerFactory.getLogger(CacheManager.class);
private static final Set<Class<? extends BaseModel>> GROUPED_CLASSES =
Set.of(Attribute.class, Device.class, Driver.class, Geofence.class, Maintenance.class, Notification.class);
private final Config config;
private final Storage storage;
private final BroadcastService broadcastService;
private final CacheGraph graph = new CacheGraph();
private volatile Server server;
private final Map<Long, ConcurrentLinkedDeque<Position>> devicePositions = new ConcurrentHashMap<>();
private final Map<Long, HashSet<Object>> deviceReferences = new ConcurrentHashMap<>();
private final Map<Long, ReentrantLock> deviceLocks = new ConcurrentHashMap<>();
@Inject
public CacheManager(Config config, Storage storage, BroadcastService broadcastService) throws StorageException {
this.config = config;
this.storage = storage;
this.broadcastService = broadcastService;
server = storage.getObject(Server.class, new Request(new Columns.All()));
// PRE-WARM: load shared objects into graph at startup
// Prevent DB query inside synchronized block when device connect
LOGGER.info("Pre-warming cache with shared objects...");
List<Class<? extends BaseModel>> sharedClasses = Arrays.asList(
Group.class, Geofence.class, Attribute.class,
Driver.class, Maintenance.class, Notification.class, Calendar.class);
for (Class<? extends BaseModel> clazz : sharedClasses) {
try {
for (BaseModel object : storage.getObjects(clazz, new Request(new Columns.All()))) {
graph.addObject(object);
}
LOGGER.debug("Pre-warmed {} objects for {}", clazz.getSimpleName(), clazz.getSimpleName());
} catch (Exception e) {
LOGGER.warn("Failed to pre-warm {}: {}", clazz.getSimpleName(), e.getMessage());
}
}
LOGGER.info("Cache pre-warming complete");
broadcastService.registerListener(this);
}
@Override
public String toString() {
return graph.toString();
}
public Config getConfig() {
return config;
}
public <T extends BaseModel> T getObject(Class<T> clazz, long id) {
return graph.getObject(clazz, id);
}
public <T extends BaseModel> Set<T> getDeviceObjects(long deviceId, Class<T> clazz) {
return graph.getObjects(Device.class, deviceId, clazz, Set.of(Group.class), true)
.collect(Collectors.toUnmodifiableSet());
}
public Position getPosition(long deviceId) {
var positions = devicePositions.get(deviceId);
return positions != null ? positions.peekLast() : null;
}
public Deque<Position> getPositions(long deviceId) {
return devicePositions.computeIfAbsent(deviceId, k -> new ConcurrentLinkedDeque<>());
}
public Server getServer() {
return server;
}
public Set<User> getNotificationUsers(long notificationId, long deviceId) {
Set<User> deviceUsers = getDeviceObjects(deviceId, User.class);
return graph.getObjects(Notification.class, notificationId, User.class, Set.of(), false)
.filter(deviceUsers::contains)
.collect(Collectors.toUnmodifiableSet());
}
public Set<Notification> getDeviceNotifications(long deviceId) {
var direct = graph.getObjects(Device.class, deviceId, Notification.class, Set.of(Group.class), true)
.map(BaseModel::getId)
.collect(Collectors.toUnmodifiableSet());
return graph.getObjects(Device.class, deviceId, Notification.class, Set.of(Group.class, User.class), true)
.filter(notification -> notification.getAlways() || direct.contains(notification.getId()))
.collect(Collectors.toUnmodifiableSet());
}
public void addDevice(long deviceId, Object key) throws Exception {
// Per-device lock: device different will not block each other
ReentrantLock deviceLock = deviceLocks.computeIfAbsent(deviceId, k -> new ReentrantLock());
deviceLock.lock();
try {
var references = deviceReferences.computeIfAbsent(deviceId, k -> new HashSet<>());
// Fast path: device already cached
if (!references.isEmpty()) {
references.add(key);
LOGGER.debug("Cache add device {} references {} key {} (fast path)", deviceId, references.size(), key);
return;
}
// Slow path: fetch ALL data from DB before acquire global lock
Device device = storage.getObject(Device.class, new Request(
new Columns.All(), new Condition.Equals("id", deviceId)));
if (device == null) {
LOGGER.warn("Device {} not found in database", deviceId);
references.add(key);
return;
}
Position position = null;
if (device.getPositionId() > 0) {
position = storage.getObject(Position.class, new Request(
new Columns.All(),
new Condition.And(
new Condition.Equals("deviceId", deviceId),
new Condition.Equals("id", device.getPositionId()))));
}
// Tandai device sebagai loading (add ke references) dalam lock singkat
boolean needsInit;
synchronized (this) {
var refs2 = deviceReferences.computeIfAbsent(deviceId, k -> new HashSet<>());
needsInit = refs2.isEmpty();
if (needsInit) {
// Add device to graph WITHOUT initializeCache (skip DB query)
graph.addObject(device);
if (position != null) {
var positions = devicePositions.computeIfAbsent(deviceId, k -> new ConcurrentLinkedDeque<>());
positions.add(position);
}
}
refs2.add(key);
LOGGER.debug("Cache add device {} references {} key {}", deviceId, refs2.size(), key);
}
// initializeCache outside lock
// Per-device lock make sure just 1 thread per device that goes to here
if (needsInit) {
initializeCache(device);
}
} finally {
deviceLock.unlock();
}
}
public void removeDevice(long deviceId, Object key) {
ReentrantLock deviceLock = deviceLocks.computeIfAbsent(deviceId, k -> new ReentrantLock());
deviceLock.lock();
try {
synchronized (this) {
var references = deviceReferences.computeIfAbsent(deviceId, k -> new HashSet<>());
references.remove(key);
if (references.isEmpty()) {
graph.removeObject(Device.class, deviceId);
devicePositions.remove(deviceId);
deviceReferences.remove(deviceId);
deviceLocks.remove(deviceId);
}
LOGGER.debug("Cache remove device {} references {} key {}", deviceId, references.size(), key);
}
} finally {
deviceLock.unlock();
}
}
private static boolean appendPosition(Deque<Position> positions, Position position) {
Position previous = positions.peekLast();
if (previous != null) {
if (position.getFixTime().before(previous.getFixTime())) {
return false;
}
if (position.getFixTime().equals(previous.getFixTime())) {
positions.pollLast();
}
}
positions.add(position);
return true;
}
public void updatePosition(Position position) {
deviceReferences.computeIfPresent(position.getDeviceId(), (key, oldValue) -> {
var positions = devicePositions.computeIfAbsent(key, k -> new ConcurrentLinkedDeque<>());
if (!appendPosition(positions, position)) {
return oldValue;
}
if (config.getBoolean(Keys.REPORT_TRIP_NEW_LOGIC)) {
long minDuration = AttributeUtil.lookup(
this, Keys.REPORT_TRIP_MIN_DURATION, key) * 1000;
long lastTime = position.getFixTime().getTime();
var iterator = positions.iterator();
iterator.next();
int toPrune = 0;
while (iterator.hasNext() && lastTime - iterator.next().getFixTime().getTime() >= minDuration) {
toPrune += 1;
}
while (toPrune-- > 0) {
positions.poll();
}
} else {
while (positions.size() > 1) {
positions.poll();
}
}
return oldValue;
});
}
@Override
public <T extends BaseModel> void invalidateObject(
boolean local, Class<T> clazz, long id, ObjectOperation operation) throws Exception {
if (local) {
broadcastService.invalidateObject(true, clazz, id, operation);
}
// Handle DELETE - quick, no DB query needed
if (operation == ObjectOperation.DELETE) {
synchronized (this) {
graph.removeObject(clazz, id);
}
return;
}
if (operation != ObjectOperation.UPDATE) {
return;
}
// Prefetch updated object OUTSIDE lock to avoid holding lock during DB query
if (clazz.equals(Server.class)) {
T updatedServer = storage.getObject(clazz, new Request(new Columns.All()));
synchronized (this) {
server = (Server) updatedServer;
}
return;
}
// Fetch updated object outside lock
var after = storage.getObject(clazz, new Request(
new Columns.All(), new Condition.Equals("id", id)));
if (after == null) {
return;
}
// Now acquire lock only for graph operations (fast, no DB)
synchronized (this) {
var before = getObject(after.getClass(), after.getId());
if (before == null) {
return;
}
switch (after) {
case GroupedModel afterGrouped -> {
long beforeGroupId = ((GroupedModel) before).getGroupId();
long afterGroupId = afterGrouped.getGroupId();
if (beforeGroupId != afterGroupId) {
if (beforeGroupId > 0) {
invalidatePermission(clazz, id, Group.class, beforeGroupId, false);
}
if (afterGroupId > 0) {
invalidatePermission(clazz, id, Group.class, afterGroupId, true);
}
}
}
case Schedulable afterSchedulable -> {
long beforeCalendarId = ((Schedulable) before).getCalendarId();
long afterCalendarId = afterSchedulable.getCalendarId();
if (beforeCalendarId != afterCalendarId) {
if (beforeCalendarId > 0) {
invalidatePermission(clazz, id, Calendar.class, beforeCalendarId, false);
}
if (afterCalendarId > 0) {
invalidatePermission(clazz, id, Calendar.class, afterCalendarId, true);
}
}
}
default -> {}
}
graph.updateObject(after);
}
}
@Override
public <T1 extends BaseModel, T2 extends BaseModel> void invalidatePermission(
boolean local, Class<T1> clazz1, long id1, Class<T2> clazz2, long id2, boolean link) throws Exception {
if (local) {
broadcastService.invalidatePermission(true, clazz1, id1, clazz2, id2, link);
}
synchronized (this) {
if (clazz1.equals(User.class) && GroupedModel.class.isAssignableFrom(clazz2)) {
invalidatePermission(clazz2, id2, clazz1, id1, link);
} else {
invalidatePermission(clazz1, id1, clazz2, id2, link);
}
}
}
private void invalidatePermission(
Class<? extends BaseModel> fromClass, long fromId,
Class<? extends BaseModel> toClass, long toId, boolean link) throws Exception {
if (toClass.equals(LinkedDevice.class)) {
toClass = Device.class;
}
boolean groupLink = GroupedModel.class.isAssignableFrom(fromClass) && toClass.equals(Group.class);
boolean calendarLink = Schedulable.class.isAssignableFrom(fromClass) && toClass.equals(Calendar.class);
boolean userLink = fromClass.equals(User.class) && toClass.equals(Notification.class);
boolean groupedLinks = GroupedModel.class.isAssignableFrom(fromClass)
&& (GROUPED_CLASSES.contains(toClass) || toClass.equals(User.class));
if (!groupLink && !calendarLink && !userLink && !groupedLinks) {
return;
}
if (link) {
if (!graph.addLink(fromClass, fromId, toClass, toId, createObjectSupplier(toClass, toId))) {
initializeCache(graph.getObject(toClass, toId));
}
} else {
graph.removeLink(fromClass, fromId, toClass, toId);
}
}
private void initializeCache(BaseModel object) throws Exception {
if (object instanceof User) {
for (Permission permission : storage.getPermissions(User.class, object.getId(), Notification.class, 0)) {
invalidatePermission(
permission.getOwnerClass(), permission.getOwnerId(),
permission.getPropertyClass(), permission.getPropertyId(), true);
}
} else {
if (object instanceof GroupedModel groupedModel) {
long groupId = groupedModel.getGroupId();
if (groupId > 0) {
invalidatePermission(object.getClass(), object.getId(), Group.class, groupId, true);
}
for (Permission permission : storage.getPermissions(User.class, 0, object.getClass(), object.getId())) {
invalidatePermission(
object.getClass(), object.getId(), User.class, permission.getOwnerId(), true);
}
for (Class<? extends BaseModel> clazz : GROUPED_CLASSES) {
if (!clazz.equals(Device.class) || object.getClass().equals(Device.class)) {
for (Permission permission : storage.getPermissions(object.getClass(), object.getId(), clazz, 0)) {
invalidatePermission(
object.getClass(), object.getId(),
clazz, permission.getPropertyId(), true);
}
}
}
}
if (object instanceof Schedulable schedulable) {
long calendarId = schedulable.getCalendarId();
if (calendarId > 0) {
invalidatePermission(object.getClass(), object.getId(), Calendar.class, calendarId, true);
}
}
}
}
private <T> Supplier<T> createObjectSupplier(Class<T> clazz, long id) {
return () -> {
try {
return storage.getObject(clazz, new Request(
new Columns.All(), new Condition.Equals("id", id)));
} catch (StorageException e) {
throw new RuntimeException(e);
}
};
}
}
That's a very dangerous change. Cache contains not only devices, but it can contain other shared entities, which can easily completely break cache. Pre-warming is completely broken, from what I can tell. I think this is a wrong direction. You need to figure out why locking takes so much time in your case. We have very large scale server without any issues. I suspect you have some database query performance problem.
Hi,
I found some issue at traccar v6.14.4 when the devices larger than 500 (currently more than 2000 devices).
below are the command to check the CLOSE-WAIT
When the CLOSE-WAIT become large like that, traccar just process authentication message only but not do decode process, it make the new positions not filled into tc_positions.
I check using jstack, if block at below
Below are my traccar.xml config
My VPS spec are
Traccar
MySQL
can anybody kindly advice?